Comparing GPT with Waterloo
load("../data/processed/networks/inter-team/2015_simplified_networks_two-step_gpt-3.5-turbo-16k_10_0-3_0-3.RData")
library(igraph)
library(dplyr)
library(ggplot2)
library(plotly)
library(wesanderson)
library(harrypotter)
source("custom_functions.R")
meta = read.csv("../data/raw/team_meta.csv", stringsAsFactors = FALSE)
#interactions = read.csv("../data/processed/curated/fuzzy_select_inter_team_raw/interactions_curated_filtered.csv", stringsAsFactors = FALSE)
#interactions = interactions %>% select(-c("category")) %>% group_by(team_matched, target_matched, year) %>% summarise(count = n())
#collab_gpt = interactions[, c("team_matched", "target_matched", "year", "count")]
#colnames(collab_gpt) = c("Source", "Target", "Year", "NbOfMention")
#collab_fuzzy = read.csv("../data/processed/collab_final.csv", stringsAsFactors = FALSE)
#collab_fuzzy = collab_fuzzy[, c("Source", "Target", "Year", "NbOfMention", "SourceID", "TargetID")]
#collab_waterloo_15 = read.csv("../data/raw/waterloo_2015_collaboration_dataset.csv", stringsAsFactors = FALSE)
#collab_waterloo_15 = collab_waterloo_15[,c("Donor", "Recipient")]
#colnames(collab_waterloo_15) = c("Source", "Target")
#collab_waterloo_15$Year = 2015
meta$Track[is.na(meta$Track)] = "Not specified"
gbh_palette = wes_palette("GrandBudapest2")
vir = viridis::viridis(6)
color_palette = c("grey", vir[2:6])
Network approach to evaluate precision/recall
eval_precision_recall = function(predicted_edges, actual_edges)
{
g1 = graph_from_data_frame(predicted_edges, directed = FALSE)
g1 = simplify(g1)
g2 = graph_from_data_frame(actual_edges, directed = FALSE)
g2 = simplify(g2)
g_diff = graph.difference(g1,g2)
pr = (ecount(g1)-ecount(g_diff))/ecount(g1)
rc = (ecount(g1)-ecount(g_diff))/ecount(g2)
return(data.frame(precision = pr, recall = rc))
}
teams_15 = meta$Team[meta$Year == "2015"]
g_gpt_15 = g_list_gpt[["2015"]]
g_fuzzy_15 = g_list_fuzzy[["2015"]]
g_waterloo_15 = g_waterloo_15
g_diff_gpt_waterloo = difference(g_gpt_15, g_waterloo_15)
precision_gw = (ecount(g_gpt_15) - ecount(g_diff_gpt_waterloo))/ecount(g_gpt_15)
recall_gw = (ecount(g_gpt_15) - ecount(g_diff_gpt_waterloo))/ecount(g_waterloo_15)
g_diff_fuzzy_waterloo = difference(g_fuzzy_15, g_waterloo_15)
precision_fw = (ecount(g_fuzzy_15) - ecount(g_diff_fuzzy_waterloo))/ecount(g_fuzzy_15)
recall_fw = (ecount(g_fuzzy_15) - ecount(g_diff_fuzzy_waterloo))/ecount(g_waterloo_15)
Jackknife Estimates
df_gpt = as_long_data_frame(g_gpt_15)[,c("from_name", "to_name")]
df_fuzzy = as_long_data_frame(g_fuzzy_15)[,c("from_name", "to_name")]
df_waterloo = as_long_data_frame(g_waterloo_15)[,c("from_name", "to_name")]
df_jkn_values = data.frame()
#Jackknife method - each 'i' is a node in the waterloo network. Removing all edges of said node from waterloo, gpt and fuzzy networks, creating a subsample
for (name in V(g_waterloo_15)$name)
{
waterloo_jkn = df_waterloo[!df_waterloo$from_name %in% name & !df_waterloo$to_name %in% name,]
df1 = eval_precision_recall(df_gpt[!df_gpt$from_name %in% name & !df_gpt$to_name %in% name,], waterloo_jkn)
df1$type = "gpt"
df1$node_removed = name
df2 = eval_precision_recall(df_fuzzy[!df_fuzzy$from_name %in% name & !df_fuzzy$to_name %in% name,], waterloo_jkn)
df2$type = "fuzzy"
df2$node_removed = name
df_jkn_values = rbind(df_jkn_values, df1)
df_jkn_values = rbind(df_jkn_values, df2)
}
jackknife_se = function(estimates, actual)
{
se = sqrt(sum((estimates-actual)**2)*((length(estimates)-1)/length(estimates)))
return(se)
}
df_errb = data.frame()
r1 = jackknife_se(df_jkn_values$recall[df_jkn_values$type == "gpt"], recall_gw)
r2 = jackknife_se(df_jkn_values$recall[df_jkn_values$type == "fuzzy"], recall_fw)
df_errb = rbind(df_errb, data.frame(model = "gpt-3.5-turbo-16k", comparison_against = "waterloo", measure = "recall", value = recall_gw, se = r1))
df_errb = rbind(df_errb, data.frame(model = "fuzzy-matching", comparison_against = "waterloo", measure = "recall", value = recall_fw, se = r2))
df_errb$model = factor(df_errb$model, levels = c("fuzzy-matching", "gpt-3.5-turbo-16k"))
plt = ggplot(df_errb) +
geom_bar(aes(fill = model, y=value, x= measure), position="dodge", stat="identity") +
scale_fill_manual(breaks = c("fuzzy-matching", "gpt-3.5-turbo-16k"), values = c(color_palette[c(1,3)])) +
geom_errorbar(aes(fill = model, ymin = value-se, ymax = value+se, x = measure), width = 0.1, position = position_dodge(0.9)) +
ggtitle("") +
theme_bw(base_size = 25) + coord_cartesian(ylim = c(0.5, 1)) +
xlab("") + ylab("") + theme(panel.grid.major = element_blank(),
panel.grid.minor = element_blank())
Warning: Ignoring unknown aesthetics: fill
ggsave(plt, filename = "../figures/precision_recall_waterloo.png", width = 6, height = 3)
ROC Curves
A_gpt <- as_adjacency_matrix(g_gpt_15, attr = "weight", sparse = FALSE)
A_fuzzy <- as_adjacency_matrix(g_fuzzy_15, attr = "weight", sparse = FALSE)
A_waterloo <- as_adjacency_matrix(g_waterloo_15, sparse = FALSE)
x_gpt <- as.numeric(A_gpt[upper.tri(A_gpt)])
x_fuzzy = as.numeric(A_fuzzy[upper.tri(A_fuzzy)])
y <- as.numeric(A_waterloo[upper.tri(A_waterloo)])
#x_gpt <- as.numeric(A_gpt)
#x_fuzzy = as.numeric(A_fuzzy)
#y <- as.numeric(A_waterloo)
r_gpt <- roc(y ~ x_gpt)
Setting levels: control = 0, case = 1
Setting direction: controls < cases
r_fuzzy <- roc(y ~ x_fuzzy)
Setting levels: control = 0, case = 1
Setting direction: controls < cases
#print(r)
df = data.frame(specificity = 1-r_gpt$specificities, sensitivity = r_gpt$sensitivities, type = "gpt-3.5-turbo-16k", thresholds = r_gpt$thresholds)
df = rbind(df, data.frame(specificity = 1-r_fuzzy$specificities, sensitivity = r_fuzzy$sensitivities, type = "fuzzy-matching", thresholds = r_fuzzy$thresholds))
df$sum =df$specificity + df$sensitivity
plt = ggplot(df, aes(x = specificity, y = sensitivity, group = type, color = type)) + geom_line() + geom_point() + geom_abline(linetype = "dashed", color = "black", alpha = 1) + xlab("") + ylab("") + xlim(c(0,1)) + scale_color_manual(breaks = c("fuzzy-matching", "gpt-3.5-turbo-16k"), values = color_palette[c(1,3)]) + theme_bw(base_size = 20) + ggtitle("") + theme(panel.grid.major = element_blank(),
panel.grid.minor = element_blank()) + theme(legend.position="none")
#+ scale_x_log10()
ggplotly(plt)
ggsave(plt, filename = "../figures/ROC_gpt_fuzzy_waterloo.png", width = 5, height = 5)
PR Curves
(Marc’s version)
library(ROCR)
pred_fuzzy <- prediction(log10(x_fuzzy+1), y >0)
pred_gpt <- prediction(log10(x_gpt+1), y >0)
perf_fuzzy <- performance(pred_fuzzy, "f")
perf_gpt <- performance(pred_gpt, "f")
# x <- log10(perf_fuzzy@x.values+1)
plot(perf_fuzzy, col=3, lwd=2)
plot(perf_gpt, add = TRUE, col=2,lwd=2)
legend('topright',
c('Fuzzy','GPT'),
col=c(3,2),
lwd=3,
bty='n',
lty=1)

# abline(h=0.5,lty=2)
# abline(0,1,lty=2)
pred_fuzzy <- prediction(log10(x_fuzzy+1), y >0)
pred_gpt <- prediction(log10(x_gpt+1), y >0)
perf_fuzzy <- performance(pred_fuzzy, "prec", "rec" )
perf_gpt <- performance(pred_gpt, "prec", "rec")
plot(perf_fuzzy, col=3, lwd=2)
plot(perf_gpt, add = TRUE, col=2,lwd=2)
legend('topright',
c('Fuzzy','GPT'),
col=c(3,2),
lwd=3,
bty='n',
lty=1)
hr <- mean(y>0)
abline(h=hr,lty=2)

pr_gpt = PRROC::pr.curve(scores.class0 = x_gpt, weights.class0 = y, curve = TRUE)$curve
pr_gpt = data.frame(pr_gpt)
pr_fuzzy = PRROC::pr.curve(scores.class0 = x_fuzzy, weights.class0 = y, curve = TRUE)$curve
pr_fuzzy = data.frame(pr_fuzzy)
#plt = ggplot() + geom_line(data = pr_gpt, aes(x = X1, y = X2), color = gbh_palette[1]) + theme_bw(base_size = 20) + geom_line(data = pr_fuzzy, aes(x = X1, y = X2), color = gbh_palette[2]) + xlab("") + ylab("")
plt = ggplot() + geom_line(data = pr_gpt, aes(x = X1, y = X2), color = color_palette[3]) + geom_line(data = pr_fuzzy, aes(x = X1, y = X2), color = color_palette[1]) + xlab("") + ylab("") + xlim(c(0,1)) + theme_bw(base_size = 20) + ggtitle("") + theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank()) + theme(legend.position="none")
#+ scale_x_log10()
#+ geom_point(data = pr_gpt, aes(x = X1, y = X2), color = color_palette[3]) + geom_point(data = pr_fuzzy, aes(x = X1, y = X2), color = color_palette[1])
#ggplotly(plt)
ggsave(plt, filename = "../figures/pr_curve_gpt_fuzzy_waterloo.png", height = 5, width = 5)
Network Properties - GPT v Fuzzy v Waterloo
populate_local_network_props = function(g)
{
deg = log(degree(g, mode = "all"), 10)
deg[is.infinite(deg)] = NA#min(deg[!is.infinite(deg)])
bet = log(betweenness(g, directed = FALSE),10)
bet[is.infinite(bet)] = NA#min(bet[!is.infinite(bet)])
eig = log(eigen_centrality(g, directed = FALSE)$vector,10)
eig[is.infinite(eig)] = NA#min(eig[!is.infinite(eig)])
clo = closeness(g, mode = "all")
clo[is.infinite(clo)] = NA#min(clo[!is.infinite(clo)])
core = coreness(g)
core[is.infinite((core))] = NA#min(core[!is.infinite(core)])
trans = transitivity(g, type = "local")
trans[is.infinite((trans))] = NA#min(trans[!is.infinite(trans)])
df = data.frame(user = V(g)$name,
deg = deg,
deg_Z = (deg-mean(deg, na.rm = TRUE))/sd(deg, na.rm = TRUE),
bet = bet,
bet_Z = (bet-mean(bet, na.rm = TRUE))/sd(bet, na.rm = TRUE),
clo = clo,
clo_Z = (clo-mean(clo, na.rm = TRUE))/sd(clo, na.rm = TRUE),
eig = eig,
eig_Z = (eig-mean(eig, na.rm = TRUE))/sd(eig, na.rm = TRUE),
core = core,
core_Z = (core-mean(core, na.rm = TRUE))/sd(core, na.rm = TRUE),
trans = trans,
trans_Z = (trans-mean(trans, na.rm = TRUE))/sd(trans, na.rm = TRUE))
return(df)
}
populate_global_network_props = function(g)
{
df = data.frame()
#web = as_adjacency_matrix(g, type = "both", sparse = FALSE)
#country = assortment.discrete(graph = web, types = as.factor(V(g)$Country), SE = TRUE)
df = rbind(df, data.frame(type = "edges", score = ecount(g)))
df = rbind(df, data.frame(type = "density", score = edge_density(g)))
df = rbind(df, data.frame(type = "triangles", score = length(triangles(g))))
df = rbind(df, data.frame(type = "connected components", score = components(g)$no))
df = rbind(df, data.frame(type = "assortativity_region", score = assortativity_nominal(g, directed = FALSE, types = as.factor(V(g)$Region))))
df = rbind(df, data.frame(type = "assortativity_country", score = assortativity_nominal(g, directed = FALSE, types = as.factor(V(g)$Country))))
#df = rbind(df, data.frame(type = "assortativity_track", score = assortativity_nominal(g, directed = FALSE, types = as.factor(V(g)$Track))))
df = rbind(df, data.frame(type = "assortativity_section", score = assortativity_nominal(g, directed = FALSE, types = as.factor(V(g)$Section))))
df = rbind(df, data.frame(type = "assortativity_size", score = assortativity(g, directed = FALSE, types1 = V(g)$Size)))
df = rbind(df, data.frame(type = "assortativity_degree", score = assortativity(g, directed = FALSE, types1 = degree(g))))
df = rbind(df, data.frame(type = "assortativity_prior success", score = assortativity_nominal(g, directed = FALSE, types = as.factor(V(g)$gold_prev_year))))
df = rbind(df, data.frame(type = "assortativity_prior experience", score = assortativity(g, directed = FALSE, types1 = V(g)$nb_participation)))
return(df)
}
Local Network Properties
df_lnp = data.frame()
df = populate_local_network_props(g_waterloo_15)
df = reshape2::melt(df, id.var = "user")
df$year = 2015
df$type = "waterloo"
df_lnp = rbind(df_lnp, df)
df = populate_local_network_props(g_gpt_15)
df = reshape2::melt(df, id.var = "user")
df$year = 2015
df$type = "gpt-3.5-turbo-16k"
df_lnp = rbind(df_lnp, df)
df = populate_local_network_props(g_fuzzy_15)
df = reshape2::melt(df, id.var = "user")
df$year = 2015
df$type = "fuzzy-matching"
df_lnp = rbind(df_lnp, df)
Scatterplots
renames = list("deg" = "Log10 Degree", "deg_Z" = "Log10 Degree Z-scored", "clo" = "Closeness", "clo_Z" = "Closeness Z-scored", "bet" = "Log10 Betweenness", "bet_Z" = "Log10 Betweenness Z-scored", "eig" = "Log10 Eigen Centrality", "eig_Z" = "Log10 Eigen Cent. Z-scored", "trans" = "Local Clustering", "trans_Z" = "Local Clustering Z-scored", "core" = "Coreness", "core_Z" = "Core Z-scored")
pdf("../figures/scatterplots_gpt_fuzzy_waterloo.pdf", height = 7, width = 7)
for (i in unique(df_lnp$variable))
{
temp = df_lnp[df_lnp$variable == i,]
temp = reshape2::dcast(temp, formula = user+year~type, value.var = "value")
plt = ggplot(temp) +
geom_point(aes(x = waterloo, y = `gpt-3.5-turbo-16k`), alpha = 0.8, color = color_palette[3]) + geom_abline(color = "black", alpha = 1) +
geom_point(aes(x = waterloo, y = `fuzzy-matching`), alpha = 0.8, color = color_palette[1]) +
ggtitle("") +
theme_bw(base_size = 20) +
xlab("Waterloo") + ylab("Predicted") + ggtitle(paste(renames[i], sep = "")) + theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1))
print(plt)
}
dev.off()
null device
1
Clustered Scatterplots (TODO)
pdf("../figures/interval_scatterplots_gpt_fuzzy_waterloo.pdf", height = 7, width = 7)
for (i in unique(df_lnp$variable))
{
temp = df_lnp[df_lnp$variable == i,]
breaks <- seq(min(temp$value[!is.na(temp$value)]), max(temp$value[!is.na(temp$value)]), length.out = 10)
classes <- cut(temp$value, breaks=breaks, include.lowest = TRUE)
temp$class = classes
#temp = reshape2::dcast(temp, formula = class~type, value.var = "value")
temp = temp %>% group_by(class, type) %>% summarise(mean = mean(value), se = se(value))
temp = reshape2::dcast(temp, formula = class~type, value.vars = c("mean"))
plt = ggplot(temp) +
geom_point(aes(x = waterloo, y = `gpt-3.5-turbo-16k`), alpha = 0.8, color = color_palette[3]) + geom_abline(color = "red", alpha = 0.3) +
geom_point(aes(x = waterloo, y = `fuzzy-matching`), alpha = 0.8, color = color_palette[1]) +
ggtitle("") +
theme_bw(base_size = 20) +
xlab("Waterloo") + ylab("Predicted") + ggtitle(paste(renames[i], sep = "")) + theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1))
print(plt)
}
`summarise()` has grouped output by 'class'. You can override using the `.groups` argument.Using se as value column: use value.var to override.
dev.off()
null device
1
Facet Wrap scatterplot
temp = reshape2::dcast(df_lnp[df_lnp$variable %in% c("deg_Z", "bet_Z", "eig_Z", "core_Z", "trans_Z"),], formula = variable+user+year~type, value.var = "value")
temp = temp %>% rowwise() %>% mutate(renamed_variable = renames[as.character(variable)][[1]])
plt = ggplot(temp) +
geom_point(aes(x = waterloo, y = `gpt-3.5-turbo-16k`), alpha = 0.6, color = color_palette[3]) + geom_abline(color = "red", alpha = 0.1) +
geom_point(aes(x = waterloo, y = `fuzzy-matching`), alpha = 0.3, color = color_palette[1]) +
ggtitle("") +
facet_wrap(~renamed_variable) +
theme_bw(base_size = 20) +
xlab("") + ylab("") + ggtitle("") + theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1))
ggsave(plt, filename = "../figures/facet_wrap_scatterplot.png", width = 10, height = 8)
Density Plot (TO edit)
pdf("../figures/local_props_density_2015.pdf", height = 7, width = 7)
for (i in unique(df_lnp$variable))
{
temp = df_lnp[df_lnp$variable == i,]
#temp = reshape2::dcast(temp, formula = user+year~type, value.var = "value")
#temp = reshape2::dcast(temp, formula = user+year~type, value.var = "value")
#plt = ggplot(temp) +
#geom_histogram(aes(x = `gpt-3.5-turbo-16k`, y = ..density..), fill=gbh_palette[1]) +
#geom_label(aes(x=4.5, y=0.25, label="GPT"), color=gbh_palette[1]) +
#geom_histogram( aes(x = `fuzzy-matching`, y = -..density..), fill= gbh_palette[2]) +
#geom_label(aes(x=4.5, y=-0.25, label="Fuzzy"), color=gbh_palette[2]) + theme_bw(base_size = 20) +
#xlab(renames[i])
plt = ggplot(temp) + geom_density(aes(x = value, group = type, fill = type, color = type), alpha = 0.8) +
ggtitle("") + scale_fill_manual(breaks = c("gpt-3.5-turbo-16k", "fuzzy-matching", "waterloo"), values = color_palette[c(1,4,6)]) +
scale_color_manual(breaks = c("gpt-3.5-turbo-16k", "fuzzy-matching", "waterloo"), values = color_palette[c(1,4,6)]) +
theme_bw(base_size = 20) +
xlab("") + ylab("") + ggtitle(paste(renames[i], sep = "")) + theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1))
print(plt)
}
dev.off()
null device
1
Degree Distribution
2015: GPT v Fuzzy v Waterloo
degree_distribution_normalised = function (graph, cumulative = FALSE)
{
if (!is_igraph(graph)) {
stop("Not a graph object")
}
cs <- degree(graph)
cs = (cs)/max(cs)
hi <- hist(cs, plot = FALSE)$density
if (!cumulative) {
res <- hi
}
else {
res <- rev(cumsum(rev(hi)))
}
return(res)
}
df_cum = data.frame()
g_gpt = g_gpt_15
g_fuzzy = g_fuzzy_15
dist = log(degree_distribution_normalised(g_gpt, cumulative = TRUE), base= 10)
df_cum = rbind(df_cum, data.frame(network = "gpt-3.5-turbo-16k", degree = 1:length(dist), degree_log = log(1:length(dist), base = 10), prob = dist, year = i))
dist = log(degree_distribution_normalised(g_fuzzy, cumulative = TRUE), base= 10)
df_cum = rbind(df_cum, data.frame(network = "fuzzy-matching", degree = 1:length(dist), degree_log = log(1:length(dist), base = 10), prob = dist, year = i))
dist = log(degree_distribution_normalised(g_waterloo_15, cumulative = TRUE), base= 10)
df_cum = rbind(df_cum, data.frame(network = "waterloo", degree = 1:length(dist), degree_log = log(1:length(dist), base = 10), prob = dist, year = i))
plt = ggplot(df_cum, aes(color = network, y=prob, x= degree_log)) +
geom_line(size = 1.2) +
scale_color_manual(breaks = c("gpt-3.5-turbo-16k", "fuzzy-matching", "waterloo"), values = color_palette[c(1,3,6)]) +
ggtitle("") +
facet_wrap(~"2015") + theme_bw(base_size = 20) +
xlab("") + ylab("") + ggtitle("")
Warning: Using `size` aesthetic for lines was deprecated in ggplot2 3.4.0.
Please use `linewidth` instead.
#theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(), panel.background = element_blank())
ggsave(plt, filename = "../figures/cumulative_dd_2015.png", width = 10, height = 8)
Correlations Boxplots
Correlations 2015 - GPT v Waterloo & Fuzzy v Waterloo
df_lnp_correlations = data.frame()
for (i in unique(df_lnp$variable))
{
temp = df_lnp[df_lnp$variable == i,]
temp = reshape2::dcast(temp, formula = user+year~type, value.var = "value")
cor = cor.test(temp$`gpt-3.5-turbo-16k`, temp$waterloo, method = "spearman")
df_lnp_correlations = rbind(df_lnp_correlations, data.frame(type = "gpt-3.5-turbo-16k", variable = i, cor = cor$estimate, p_val = cor$p.value))
cor = cor.test(temp$`fuzzy-matching`, temp$waterloo, method = "spearman")
df_lnp_correlations = rbind(df_lnp_correlations, data.frame(type = "fuzzy-matching", variable = i, cor = cor$estimate, p_val = cor$p.value))
}
Boxplots (TO CHECK)
plt = ggplot(df_lnp_correlations, aes(x = cor, y = variable)) +
geom_bar(stat = "identity") + scale_y_discrete(labels = renames) +
ggtitle("") +
theme_bw(base_size = 20) +
xlab("Spearman Correlations") + ylab("") + ggtitle("")
#ggsave(plt, filename = "../figures/correlations_years.png", height = 7, width = 9)
Global Network Properties
df_gnp = data.frame()
#GPT
df = populate_global_network_props(g_gpt_15)
df$model = "gpt-3.5-turbo-16k"
df_gnp = rbind(df_gnp, df)
#fuzzy
df = populate_global_network_props(g_fuzzy_15)
df$model = "fuzzy-matching"
df_gnp = rbind(df_gnp, df)
#waterloo
df = populate_global_network_props(g_waterloo_15)
df$model = "waterloo"
df_gnp = rbind(df_gnp, df)
Bar Plot (minus jackknife)
plt = ggplot(df_gnp[!df_gnp$type %in% c("edges", "triangles", "connected components"),]) + geom_bar(aes(x = type, y = score, fill = model), stat = "identity", position = position_dodge()) + theme_bw(base_size = 25) + scale_fill_manual(breaks = c("gpt-3.5-turbo-16k", "fuzzy-matching", "waterloo"), values = color_palette[c(1,4,6)]) + theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1))
Correlations with Jackknifes
(LOCAL)
#
# df_lcor_jkn = data.frame()
#
# for (name in V(g_waterloo_15)$name)
# {
# temp = data.frame()
#
# g_gpt = delete.vertices(g_gpt_15, v = name)
# g_fuzzy = delete.vertices(g_fuzzy_15, v = name)
# g_waterloo = delete.vertices(g_waterloo_15, v = name)
#
# df = populate_local_network_props(g_gpt)
# df = reshape2::melt(df, id.var = "user")
# df$model = "gpt-3.5-turbo-16k"
# temp = rbind(temp, df)
#
# df = populate_local_network_props(g_fuzzy)
# df = reshape2::melt(df, id.var = "user")
# df$model = "fuzzy-matching"
# temp = rbind(temp, df)
#
# df = populate_local_network_props(g_waterloo)
# df = reshape2::melt(df, id.var = "user")
# df$model = "waterloo"
# temp = rbind(temp, df)
#
# temp$removed = name
#
# df_lcor_jkn = rbind(df_lcor_jkn, temp)
#
# }
Compute Errorbars with Jackknife
# temp = df_lcor_jkn
# temp = reshape2::dcast(temp, formula = user+removed+variable~model, value.var = "value")
#
# t = temp %>% group_by(removed, variable) %>% summarise(cor_gpt = cor.test(waterloo, `gpt-3.5-turbo-16k`, method = "spearman")$estimate, cor_fuzzy = cor.test(waterloo, `fuzzy-matching`, method = "spearman")$estimate)
df_cor_jkn = data.frame()
for (i in unique(df_lnp$variable))
{
temp = df_lnp[df_lnp$variable == i,]
temp = reshape2::dcast(temp, formula = user+year~type, value.var = "value")
jkn = data.frame()
for (j in 1:nrow(temp))
{
t = temp[-c(j),]
jkn = rbind(jkn, data.frame(id = j, cor_gpt = cor.test(t$`gpt-3.5-turbo-16k`, t$waterloo, method = "spearman")$estimate, cor_fuzzy = cor.test(t$`fuzzy-matching`, t$waterloo, method = "spearman")$estimate))
}
actual = cor.test(temp$`gpt-3.5-turbo-16k`, temp$waterloo, method = "spearman")$estimate
estimates = jkn$cor_gpt
df_cor_jkn = rbind(df_cor_jkn, data.frame(variable = i, value = actual, se = jackknife_se(estimates, actual), type = "gpt-3.5-turbo-16k"))
actual = cor.test(temp$`fuzzy-matching`, temp$waterloo, method = "spearman")$estimate
estimates = jkn$cor_fuzzy
df_cor_jkn = rbind(df_cor_jkn, data.frame(variable = i, value = actual, se = jackknife_se(estimates, actual), type = "fuzzy-matching"))
}
Make the bar plots
#[df_cor_jkn$variable %in% c("deg", "core", "bet", "clo", "eig"),]
renames = c("core" = "Coreness", "bet" = "Betweenness", "clo" = "Closeness", "trans" = "Clustering")
plt = ggplot(df_cor_jkn[df_cor_jkn$variable %in% c("core", "bet", "clo", "trans"),]) +
geom_bar(aes(fill = type, y=value, x=variable), position="dodge", stat="identity") +
scale_fill_manual(breaks = c("fuzzy-matching", "gpt-3.5-turbo-16k"), values = (color_palette[c(1,3)])) +
geom_errorbar(aes(fill = type, ymin = value-se, ymax = value+se, x = variable), width = 0.1, position = position_dodge(0.9)) +
ggtitle("") +
theme_bw(base_size = 25) + coord_cartesian(ylim = c(-0.25, 1)) +
xlab("") + ylab("") + theme(panel.grid.major = element_blank(),
panel.grid.minor = element_blank()) + theme(legend.position="none") + scale_x_discrete(labels = renames) + theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1))
Warning: Ignoring unknown aesthetics: fill
#ggplotly(plt)
ggsave(plt, filename = "../figures/local_net_props_gpt_fuzzy_waterloo.png", width = 4, height = 6)
(GLOBAL NETWORK PROPERTIES)
df_jkn_gnp = data.frame()
for (name in V(g_waterloo_15)$name)
{
df_gnp = data.frame()
g_gpt = delete.vertices(g_gpt_15, v = name)
g_fuzzy = delete.vertices(g_fuzzy_15, v = name)
g_waterloo = delete.vertices(g_waterloo_15, v = name)
#GPT
df = populate_global_network_props(g_gpt)
df$model = "gpt-3.5-turbo-16k"
df_gnp = rbind(df_gnp, df)
#fuzzy
df = populate_global_network_props(g_fuzzy)
df$model = "fuzzy-matching"
df_gnp = rbind(df_gnp, df)
#waterloo
df = populate_global_network_props(g_waterloo)
df$model = "waterloo"
df_gnp = rbind(df_gnp, df)
df_gnp$removed = name
df_jkn_gnp = rbind(df_jkn_gnp, df_gnp)
}
df_gnp = data.frame()
#GPT
df = populate_global_network_props(g_gpt_15)
df$model = "gpt-3.5-turbo-16k"
df_gnp = rbind(df_gnp, df)
#fuzzy
df = populate_global_network_props(g_fuzzy_15)
df$model = "fuzzy-matching"
df_gnp = rbind(df_gnp, df)
#waterloo
df = populate_global_network_props(g_waterloo_15)
df$model = "waterloo"
df_gnp = rbind(df_gnp, df)
Assortativity
df_gnp_assort = data.frame()
for (i in unique(df_gnp$type))
{
temp = df_gnp[df_gnp$type == i,]
#temp = reshape2::dcast(temp, formula = model~score, value.var = "score")
val = temp$score[temp$model == "gpt-3.5-turbo-16k"]
df_gnp_assort = rbind(df_gnp_assort, data.frame(type = "gpt-3.5-turbo-16k", variable = i, val = val, se = jackknife_se(df_jkn_gnp$score[df_jkn_gnp$model == "gpt-3.5-turbo-16k" & df_jkn_gnp$type == i], val)))
#cor = cor.test(temp$`fuzzy-matching`, temp$waterloo, method = "spearman")
val = temp$score[temp$model == "fuzzy-matching"]
df_gnp_assort = rbind(df_gnp_assort, data.frame(type = "fuzzy-matching", variable = i, val = val, se = jackknife_se(df_jkn_gnp$score[df_jkn_gnp$model == "fuzzy-matching" & df_jkn_gnp$type == i], val)))
val = temp$score[temp$model == "waterloo"]
df_gnp_assort = rbind(df_gnp_assort, data.frame(type = "waterloo", variable = i, val = val, se = jackknife_se(df_jkn_gnp$score[df_jkn_gnp$model == "waterloo" & df_jkn_gnp$type == i], val)))
}
Plot
renames = list("assortativity_country" = "Country", "assortativity_region" = "Region", "assortativity_degree" = "Degree", "assortativity_prior experience" = "Prior Experience", "assortativity_prior success" = "Success", "assortativity_size" = "Size", "connected components" = "Components", "triangles" = "Triangles", "density" = "Density", "edges" = "Edges", "assortativity_section" = "Section")
plt = ggplot(df_gnp_assort[df_gnp_assort$variable %in% c("assortativity_country", "assortativity_region", "assortativity_section", "assortativity_degree"),]) +
geom_bar(aes(fill = type, y=val, x=variable), position="dodge", stat="identity") +
scale_fill_manual(breaks = c("fuzzy-matching", "gpt-3.5-turbo-16k", "waterloo"), values = c(color_palette[c(1,3)], "yellow3")) +
geom_errorbar(aes(fill = type, ymin = val-se, ymax = val+se, x = variable), width = 0.1, position = position_dodge(0.9)) +
ggtitle("") +
theme_bw(base_size = 25) + coord_cartesian(ylim = c(-0.25, 1)) +
xlab("") + ylab("") + theme(panel.grid.major = element_blank(),
panel.grid.minor = element_blank()) + theme(legend.position="none") + scale_x_discrete(labels = renames) + theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) + ggtitle("")
Warning: Ignoring unknown aesthetics: fill
ggsave(plt, filename = "../figures/assortativity_2015.png", width = 5, height = 6)
LS0tCnRpdGxlOiAiUiBOb3RlYm9vayIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKQ29tcGFyaW5nIEdQVCB3aXRoIFdhdGVybG9vCgpgYGB7ciwgd2FybmluZz1GQUxTRSwgbWVzc2FnZT1GQUxTRX0KCmxvYWQoIi4uL2RhdGEvcHJvY2Vzc2VkL25ldHdvcmtzL2ludGVyLXRlYW0vMjAxNV9zaW1wbGlmaWVkX25ldHdvcmtzX3R3by1zdGVwX2dwdC0zLjUtdHVyYm8tMTZrXzEwXzAtM18wLTMuUkRhdGEiKQoKbGlicmFyeShpZ3JhcGgpCmxpYnJhcnkoZHBseXIpCmxpYnJhcnkoZ2dwbG90MikKbGlicmFyeShwbG90bHkpCmxpYnJhcnkod2VzYW5kZXJzb24pCmxpYnJhcnkoaGFycnlwb3R0ZXIpCgpzb3VyY2UoImN1c3RvbV9mdW5jdGlvbnMuUiIpCgptZXRhID0gcmVhZC5jc3YoIi4uL2RhdGEvcmF3L3RlYW1fbWV0YS5jc3YiLCBzdHJpbmdzQXNGYWN0b3JzID0gRkFMU0UpCgojaW50ZXJhY3Rpb25zID0gcmVhZC5jc3YoIi4uL2RhdGEvcHJvY2Vzc2VkL2N1cmF0ZWQvZnV6enlfc2VsZWN0X2ludGVyX3RlYW1fcmF3L2ludGVyYWN0aW9uc19jdXJhdGVkX2ZpbHRlcmVkLmNzdiIsIHN0cmluZ3NBc0ZhY3RvcnMgPSBGQUxTRSkKI2ludGVyYWN0aW9ucyA9IGludGVyYWN0aW9ucyAlPiUgc2VsZWN0KC1jKCJjYXRlZ29yeSIpKSAlPiUgZ3JvdXBfYnkodGVhbV9tYXRjaGVkLCB0YXJnZXRfbWF0Y2hlZCwgeWVhcikgJT4lIHN1bW1hcmlzZShjb3VudCA9IG4oKSkKCiNjb2xsYWJfZ3B0ID0gaW50ZXJhY3Rpb25zWywgYygidGVhbV9tYXRjaGVkIiwgInRhcmdldF9tYXRjaGVkIiwgInllYXIiLCAiY291bnQiKV0KI2NvbG5hbWVzKGNvbGxhYl9ncHQpID0gYygiU291cmNlIiwgIlRhcmdldCIsICJZZWFyIiwgIk5iT2ZNZW50aW9uIikKCiNjb2xsYWJfZnV6enkgPSByZWFkLmNzdigiLi4vZGF0YS9wcm9jZXNzZWQvY29sbGFiX2ZpbmFsLmNzdiIsIHN0cmluZ3NBc0ZhY3RvcnMgPSBGQUxTRSkKI2NvbGxhYl9mdXp6eSA9IGNvbGxhYl9mdXp6eVssIGMoIlNvdXJjZSIsICJUYXJnZXQiLCAiWWVhciIsICJOYk9mTWVudGlvbiIsICJTb3VyY2VJRCIsICJUYXJnZXRJRCIpXQoKI2NvbGxhYl93YXRlcmxvb18xNSA9IHJlYWQuY3N2KCIuLi9kYXRhL3Jhdy93YXRlcmxvb18yMDE1X2NvbGxhYm9yYXRpb25fZGF0YXNldC5jc3YiLCBzdHJpbmdzQXNGYWN0b3JzID0gRkFMU0UpCiNjb2xsYWJfd2F0ZXJsb29fMTUgPSBjb2xsYWJfd2F0ZXJsb29fMTVbLGMoIkRvbm9yIiwgIlJlY2lwaWVudCIpXQojY29sbmFtZXMoY29sbGFiX3dhdGVybG9vXzE1KSA9IGMoIlNvdXJjZSIsICJUYXJnZXQiKQojY29sbGFiX3dhdGVybG9vXzE1JFllYXIgPSAyMDE1CgptZXRhJFRyYWNrW2lzLm5hKG1ldGEkVHJhY2spXSA9ICJOb3Qgc3BlY2lmaWVkIgoKZ2JoX3BhbGV0dGUgPSB3ZXNfcGFsZXR0ZSgiR3JhbmRCdWRhcGVzdDIiKQp2aXIgPSB2aXJpZGlzOjp2aXJpZGlzKDYpCgpjb2xvcl9wYWxldHRlID0gYygiZ3JleSIsIHZpclsyOjZdKQoKYGBgCgpOZXR3b3JrIGFwcHJvYWNoIHRvIGV2YWx1YXRlIHByZWNpc2lvbi9yZWNhbGwKCmBgYHtyfQoKZXZhbF9wcmVjaXNpb25fcmVjYWxsID0gZnVuY3Rpb24ocHJlZGljdGVkX2VkZ2VzLCBhY3R1YWxfZWRnZXMpCnsKICAKICBnMSA9IGdyYXBoX2Zyb21fZGF0YV9mcmFtZShwcmVkaWN0ZWRfZWRnZXMsIGRpcmVjdGVkID0gRkFMU0UpCiAgZzEgPSBzaW1wbGlmeShnMSkKICAKICBnMiA9IGdyYXBoX2Zyb21fZGF0YV9mcmFtZShhY3R1YWxfZWRnZXMsIGRpcmVjdGVkID0gRkFMU0UpCiAgZzIgPSBzaW1wbGlmeShnMikKICAKICBnX2RpZmYgPSBncmFwaC5kaWZmZXJlbmNlKGcxLGcyKQogIAogIHByID0gKGVjb3VudChnMSktZWNvdW50KGdfZGlmZikpL2Vjb3VudChnMSkKICByYyA9IChlY291bnQoZzEpLWVjb3VudChnX2RpZmYpKS9lY291bnQoZzIpCiAgCiAgcmV0dXJuKGRhdGEuZnJhbWUocHJlY2lzaW9uID0gcHIsIHJlY2FsbCA9IHJjKSkKICAKfQoKYGBgCgoKYGBge3J9Cgp0ZWFtc18xNSA9IG1ldGEkVGVhbVttZXRhJFllYXIgPT0gIjIwMTUiXQoKZ19ncHRfMTUgPSBnX2xpc3RfZ3B0W1siMjAxNSJdXQpnX2Z1enp5XzE1ID0gZ19saXN0X2Z1enp5W1siMjAxNSJdXQpnX3dhdGVybG9vXzE1ID0gZ193YXRlcmxvb18xNQoKZ19kaWZmX2dwdF93YXRlcmxvbyA9IGRpZmZlcmVuY2UoZ19ncHRfMTUsIGdfd2F0ZXJsb29fMTUpCnByZWNpc2lvbl9ndyA9IChlY291bnQoZ19ncHRfMTUpIC0gZWNvdW50KGdfZGlmZl9ncHRfd2F0ZXJsb28pKS9lY291bnQoZ19ncHRfMTUpCnJlY2FsbF9ndyA9IChlY291bnQoZ19ncHRfMTUpIC0gZWNvdW50KGdfZGlmZl9ncHRfd2F0ZXJsb28pKS9lY291bnQoZ193YXRlcmxvb18xNSkKCmdfZGlmZl9mdXp6eV93YXRlcmxvbyA9IGRpZmZlcmVuY2UoZ19mdXp6eV8xNSwgZ193YXRlcmxvb18xNSkKcHJlY2lzaW9uX2Z3ID0gKGVjb3VudChnX2Z1enp5XzE1KSAtIGVjb3VudChnX2RpZmZfZnV6enlfd2F0ZXJsb28pKS9lY291bnQoZ19mdXp6eV8xNSkKcmVjYWxsX2Z3ID0gKGVjb3VudChnX2Z1enp5XzE1KSAtIGVjb3VudChnX2RpZmZfZnV6enlfd2F0ZXJsb28pKS9lY291bnQoZ193YXRlcmxvb18xNSkKCmBgYAoKSmFja2tuaWZlIEVzdGltYXRlcwoKYGBge3J9CgpkZl9ncHQgPSBhc19sb25nX2RhdGFfZnJhbWUoZ19ncHRfMTUpWyxjKCJmcm9tX25hbWUiLCAidG9fbmFtZSIpXQpkZl9mdXp6eSA9IGFzX2xvbmdfZGF0YV9mcmFtZShnX2Z1enp5XzE1KVssYygiZnJvbV9uYW1lIiwgInRvX25hbWUiKV0KZGZfd2F0ZXJsb28gPSBhc19sb25nX2RhdGFfZnJhbWUoZ193YXRlcmxvb18xNSlbLGMoImZyb21fbmFtZSIsICJ0b19uYW1lIildCgpkZl9qa25fdmFsdWVzID0gZGF0YS5mcmFtZSgpCiAKI0phY2trbmlmZSBtZXRob2QgLSBlYWNoICdpJyBpcyBhIG5vZGUgaW4gdGhlIHdhdGVybG9vIG5ldHdvcmsuIFJlbW92aW5nIGFsbCBlZGdlcyBvZiBzYWlkIG5vZGUgZnJvbSB3YXRlcmxvbywgZ3B0IGFuZCBmdXp6eSBuZXR3b3JrcywgY3JlYXRpbmcgYSBzdWJzYW1wbGUKCmZvciAobmFtZSBpbiBWKGdfd2F0ZXJsb29fMTUpJG5hbWUpCnsKICAgd2F0ZXJsb29famtuID0gZGZfd2F0ZXJsb29bIWRmX3dhdGVybG9vJGZyb21fbmFtZSAlaW4lIG5hbWUgJiAhZGZfd2F0ZXJsb28kdG9fbmFtZSAlaW4lIG5hbWUsXQogIAogICBkZjEgPSBldmFsX3ByZWNpc2lvbl9yZWNhbGwoZGZfZ3B0WyFkZl9ncHQkZnJvbV9uYW1lICVpbiUgbmFtZSAmICFkZl9ncHQkdG9fbmFtZSAlaW4lIG5hbWUsXSwgd2F0ZXJsb29famtuKQogICBkZjEkdHlwZSA9ICJncHQiCiAgIGRmMSRub2RlX3JlbW92ZWQgPSBuYW1lCiAgIGRmMiA9IGV2YWxfcHJlY2lzaW9uX3JlY2FsbChkZl9mdXp6eVshZGZfZnV6enkkZnJvbV9uYW1lICVpbiUgbmFtZSAmICFkZl9mdXp6eSR0b19uYW1lICVpbiUgbmFtZSxdLCB3YXRlcmxvb19qa24pCiAgIGRmMiR0eXBlID0gImZ1enp5IgogICBkZjIkbm9kZV9yZW1vdmVkID0gbmFtZQogICAKICAgZGZfamtuX3ZhbHVlcyA9IHJiaW5kKGRmX2prbl92YWx1ZXMsIGRmMSkKICAgZGZfamtuX3ZhbHVlcyA9IHJiaW5kKGRmX2prbl92YWx1ZXMsIGRmMikKICAgCn0KCgpgYGAKCmBgYHtyfQoKamFja2tuaWZlX3NlID0gZnVuY3Rpb24oZXN0aW1hdGVzLCBhY3R1YWwpCnsKICBzZSA9IHNxcnQoc3VtKChlc3RpbWF0ZXMtYWN0dWFsKSoqMikqKChsZW5ndGgoZXN0aW1hdGVzKS0xKS9sZW5ndGgoZXN0aW1hdGVzKSkpCiAgcmV0dXJuKHNlKQp9CgpkZl9lcnJiID0gZGF0YS5mcmFtZSgpCgpyMSA9IGphY2trbmlmZV9zZShkZl9qa25fdmFsdWVzJHJlY2FsbFtkZl9qa25fdmFsdWVzJHR5cGUgPT0gImdwdCJdLCByZWNhbGxfZ3cpCnIyID0gamFja2tuaWZlX3NlKGRmX2prbl92YWx1ZXMkcmVjYWxsW2RmX2prbl92YWx1ZXMkdHlwZSA9PSAiZnV6enkiXSwgcmVjYWxsX2Z3KQoKZGZfZXJyYiA9IHJiaW5kKGRmX2VycmIsIGRhdGEuZnJhbWUobW9kZWwgPSAiZ3B0LTMuNS10dXJiby0xNmsiLCBjb21wYXJpc29uX2FnYWluc3QgPSAid2F0ZXJsb28iLCBtZWFzdXJlID0gInJlY2FsbCIsIHZhbHVlID0gcmVjYWxsX2d3LCBzZSA9IHIxKSkKCmRmX2VycmIgPSByYmluZChkZl9lcnJiLCBkYXRhLmZyYW1lKG1vZGVsID0gImZ1enp5LW1hdGNoaW5nIiwgY29tcGFyaXNvbl9hZ2FpbnN0ID0gIndhdGVybG9vIiwgbWVhc3VyZSA9ICJyZWNhbGwiLCB2YWx1ZSA9IHJlY2FsbF9mdywgc2UgPSByMikpCgoKZGZfZXJyYiRtb2RlbCA9IGZhY3RvcihkZl9lcnJiJG1vZGVsLCBsZXZlbHMgPSBjKCJmdXp6eS1tYXRjaGluZyIsICJncHQtMy41LXR1cmJvLTE2ayIpKQoKcGx0ID0gZ2dwbG90KGRmX2VycmIpICsgCiAgICBnZW9tX2JhcihhZXMoZmlsbCA9IG1vZGVsLCB5PXZhbHVlLCB4PSBtZWFzdXJlKSwgcG9zaXRpb249ImRvZGdlIiwgc3RhdD0iaWRlbnRpdHkiKSArCiAgICBzY2FsZV9maWxsX21hbnVhbChicmVha3MgPSBjKCJmdXp6eS1tYXRjaGluZyIsICJncHQtMy41LXR1cmJvLTE2ayIpLCB2YWx1ZXMgPSBjKGNvbG9yX3BhbGV0dGVbYygxLDMpXSkpICsKICAgIGdlb21fZXJyb3JiYXIoYWVzKGZpbGwgPSBtb2RlbCwgeW1pbiA9IHZhbHVlLXNlLCB5bWF4ID0gdmFsdWUrc2UsIHggPSBtZWFzdXJlKSwgd2lkdGggPSAwLjEsIHBvc2l0aW9uID0gcG9zaXRpb25fZG9kZ2UoMC45KSkgKwogICAgZ2d0aXRsZSgiIikgKwogICAgdGhlbWVfYncoYmFzZV9zaXplID0gMjUpICsgY29vcmRfY2FydGVzaWFuKHlsaW0gPSBjKDAuNSwgMSkpICsgCiAgICB4bGFiKCIiKSArIHlsYWIoIiIpICsgdGhlbWUocGFuZWwuZ3JpZC5tYWpvciA9IGVsZW1lbnRfYmxhbmsoKSwKcGFuZWwuZ3JpZC5taW5vciA9IGVsZW1lbnRfYmxhbmsoKSkKCmdnc2F2ZShwbHQsIGZpbGVuYW1lID0gIi4uL2ZpZ3VyZXMvcHJlY2lzaW9uX3JlY2FsbF93YXRlcmxvby5wbmciLCB3aWR0aCA9IDYsIGhlaWdodCA9IDMpCmBgYAoKUk9DIEN1cnZlcwoKYGBge3J9CgpBX2dwdCA8LSBhc19hZGphY2VuY3lfbWF0cml4KGdfZ3B0XzE1LCBhdHRyID0gIndlaWdodCIsIHNwYXJzZSA9IEZBTFNFKQpBX2Z1enp5IDwtIGFzX2FkamFjZW5jeV9tYXRyaXgoZ19mdXp6eV8xNSwgYXR0ciA9ICJ3ZWlnaHQiLCBzcGFyc2UgPSBGQUxTRSkKQV93YXRlcmxvbyA8LSBhc19hZGphY2VuY3lfbWF0cml4KGdfd2F0ZXJsb29fMTUsIHNwYXJzZSA9IEZBTFNFKQogIAp4X2dwdCA8LSBhcy5udW1lcmljKEFfZ3B0W3VwcGVyLnRyaShBX2dwdCldKQp4X2Z1enp5ID0gYXMubnVtZXJpYyhBX2Z1enp5W3VwcGVyLnRyaShBX2Z1enp5KV0pCnkgPC0gYXMubnVtZXJpYyhBX3dhdGVybG9vW3VwcGVyLnRyaShBX3dhdGVybG9vKV0pCiAgCiN4X2dwdCA8LSBhcy5udW1lcmljKEFfZ3B0KQojeF9mdXp6eSA9IGFzLm51bWVyaWMoQV9mdXp6eSkKI3kgPC0gYXMubnVtZXJpYyhBX3dhdGVybG9vKQoKcl9ncHQgPC0gcm9jKHkgfiB4X2dwdCkKcl9mdXp6eSA8LSByb2MoeSB+IHhfZnV6enkpCiAgCiNwcmludChyKQoKZGYgPSBkYXRhLmZyYW1lKHNwZWNpZmljaXR5ID0gMS1yX2dwdCRzcGVjaWZpY2l0aWVzLCBzZW5zaXRpdml0eSA9IHJfZ3B0JHNlbnNpdGl2aXRpZXMsIHR5cGUgPSAiZ3B0LTMuNS10dXJiby0xNmsiLCB0aHJlc2hvbGRzID0gcl9ncHQkdGhyZXNob2xkcykKZGYgPSByYmluZChkZiwgZGF0YS5mcmFtZShzcGVjaWZpY2l0eSA9IDEtcl9mdXp6eSRzcGVjaWZpY2l0aWVzLCBzZW5zaXRpdml0eSA9IHJfZnV6enkkc2Vuc2l0aXZpdGllcywgdHlwZSA9ICJmdXp6eS1tYXRjaGluZyIsIHRocmVzaG9sZHMgPSByX2Z1enp5JHRocmVzaG9sZHMpKQogIApkZiRzdW0gPWRmJHNwZWNpZmljaXR5ICsgZGYkc2Vuc2l0aXZpdHkKCnBsdCA9IGdncGxvdChkZiwgYWVzKHggPSBzcGVjaWZpY2l0eSwgeSA9IHNlbnNpdGl2aXR5LCBncm91cCA9IHR5cGUsIGNvbG9yID0gdHlwZSkpICsgZ2VvbV9saW5lKCkgKyBnZW9tX3BvaW50KCkgKyBnZW9tX2FibGluZShsaW5ldHlwZSA9ICJkYXNoZWQiLCBjb2xvciA9ICJibGFjayIsIGFscGhhID0gMSkgKyB4bGFiKCIiKSArIHlsYWIoIiIpICsgeGxpbShjKDAsMSkpICsgIHNjYWxlX2NvbG9yX21hbnVhbChicmVha3MgPSBjKCJmdXp6eS1tYXRjaGluZyIsICJncHQtMy41LXR1cmJvLTE2ayIpLCB2YWx1ZXMgPSBjb2xvcl9wYWxldHRlW2MoMSwzKV0pICsgdGhlbWVfYncoYmFzZV9zaXplID0gMjApICArIGdndGl0bGUoIiIpICsgdGhlbWUocGFuZWwuZ3JpZC5tYWpvciA9IGVsZW1lbnRfYmxhbmsoKSwKcGFuZWwuZ3JpZC5taW5vciA9IGVsZW1lbnRfYmxhbmsoKSkgKyB0aGVtZShsZWdlbmQucG9zaXRpb249Im5vbmUiKQojKyBzY2FsZV94X2xvZzEwKCkKCmdncGxvdGx5KHBsdCkKCmdnc2F2ZShwbHQsIGZpbGVuYW1lID0gIi4uL2ZpZ3VyZXMvUk9DX2dwdF9mdXp6eV93YXRlcmxvby5wbmciLCB3aWR0aCA9IDUsIGhlaWdodCA9IDUpCgpgYGAKClBSIEN1cnZlcwoKKE1hcmMncyB2ZXJzaW9uKQoKYGBge3J9CgpsaWJyYXJ5KFJPQ1IpCgpwcmVkX2Z1enp5IDwtIHByZWRpY3Rpb24obG9nMTAoeF9mdXp6eSsxKSwgeSA+MCkKcHJlZF9ncHQgPC0gcHJlZGljdGlvbihsb2cxMCh4X2dwdCsxKSwgeSA+MCkKCnBlcmZfZnV6enkgPC0gcGVyZm9ybWFuY2UocHJlZF9mdXp6eSwgImYiKQpwZXJmX2dwdCA8LSBwZXJmb3JtYW5jZShwcmVkX2dwdCwgImYiKQoKIyB4IDwtIGxvZzEwKHBlcmZfZnV6enlAeC52YWx1ZXMrMSkKCnBsb3QocGVyZl9mdXp6eSwgY29sPTMsIGx3ZD0yKQpwbG90KHBlcmZfZ3B0LCBhZGQgPSBUUlVFLCBjb2w9Mixsd2Q9MikKbGVnZW5kKCd0b3ByaWdodCcsCiAgICAgICBjKCdGdXp6eScsJ0dQVCcpLAogICAgICAgICBjb2w9YygzLDIpLAogICAgICAgbHdkPTMsCiAgICAgICBidHk9J24nLCAgCiAgICAgICBsdHk9MSkKCiMgYWJsaW5lKGg9MC41LGx0eT0yKQoKIyBhYmxpbmUoMCwxLGx0eT0yKQoKYGBgCgpgYGB7cn0KCnByZWRfZnV6enkgPC0gcHJlZGljdGlvbihsb2cxMCh4X2Z1enp5KzEpLCB5ID4wKQpwcmVkX2dwdCA8LSBwcmVkaWN0aW9uKGxvZzEwKHhfZ3B0KzEpLCB5ID4wKQoKcGVyZl9mdXp6eSA8LSBwZXJmb3JtYW5jZShwcmVkX2Z1enp5LCAicHJlYyIsICJyZWMiICkKcGVyZl9ncHQgPC0gcGVyZm9ybWFuY2UocHJlZF9ncHQsICJwcmVjIiwgInJlYyIpCgpwbG90KHBlcmZfZnV6enksIGNvbD0zLCBsd2Q9MikKcGxvdChwZXJmX2dwdCwgYWRkID0gVFJVRSwgY29sPTIsbHdkPTIpCmxlZ2VuZCgndG9wcmlnaHQnLAogICAgICAgYygnRnV6enknLCdHUFQnKSwKICAgICAgICAgY29sPWMoMywyKSwKICAgICAgIGx3ZD0zLAogICAgICAgYnR5PSduJywgIAogICAgICAgbHR5PTEpCgpociA8LSBtZWFuKHk+MCkKYWJsaW5lKGg9aHIsbHR5PTIpCgpgYGAKCgoKYGBge3J9Cgpwcl9ncHQgPSBQUlJPQzo6cHIuY3VydmUoc2NvcmVzLmNsYXNzMCA9IHhfZ3B0LCB3ZWlnaHRzLmNsYXNzMCA9IHksIGN1cnZlID0gVFJVRSkkY3VydmUKcHJfZ3B0ID0gZGF0YS5mcmFtZShwcl9ncHQpCgpwcl9mdXp6eSA9IFBSUk9DOjpwci5jdXJ2ZShzY29yZXMuY2xhc3MwID0geF9mdXp6eSwgd2VpZ2h0cy5jbGFzczAgPSB5LCBjdXJ2ZSA9IFRSVUUpJGN1cnZlCnByX2Z1enp5ID0gZGF0YS5mcmFtZShwcl9mdXp6eSkKCiNwbHQgPSBnZ3Bsb3QoKSArIGdlb21fbGluZShkYXRhID0gcHJfZ3B0LCBhZXMoeCA9IFgxLCB5ID0gWDIpLCBjb2xvciA9IGdiaF9wYWxldHRlWzFdKSArIHRoZW1lX2J3KGJhc2Vfc2l6ZSA9IDIwKSAgKyBnZW9tX2xpbmUoZGF0YSA9IHByX2Z1enp5LCBhZXMoeCA9IFgxLCB5ID0gWDIpLCBjb2xvciA9IGdiaF9wYWxldHRlWzJdKSArIHhsYWIoIiIpICsgeWxhYigiIikKCnBsdCA9IGdncGxvdCgpICsgZ2VvbV9saW5lKGRhdGEgPSBwcl9ncHQsIGFlcyh4ID0gWDEsIHkgPSBYMiksIGNvbG9yID0gY29sb3JfcGFsZXR0ZVszXSkgKyBnZW9tX2xpbmUoZGF0YSA9IHByX2Z1enp5LCBhZXMoeCA9IFgxLCB5ID0gWDIpLCBjb2xvciA9IGNvbG9yX3BhbGV0dGVbMV0pICArIHhsYWIoIiIpICsgeWxhYigiIikgKyB4bGltKGMoMCwxKSkgKyB0aGVtZV9idyhiYXNlX3NpemUgPSAyMCkgICsgZ2d0aXRsZSgiIikgKyB0aGVtZShwYW5lbC5ncmlkLm1ham9yID0gZWxlbWVudF9ibGFuaygpLCBwYW5lbC5ncmlkLm1pbm9yID0gZWxlbWVudF9ibGFuaygpKSArIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbj0ibm9uZSIpCiMrIHNjYWxlX3hfbG9nMTAoKQojKyBnZW9tX3BvaW50KGRhdGEgPSBwcl9ncHQsIGFlcyh4ID0gWDEsIHkgPSBYMiksIGNvbG9yID0gY29sb3JfcGFsZXR0ZVszXSkgKyBnZW9tX3BvaW50KGRhdGEgPSBwcl9mdXp6eSwgYWVzKHggPSBYMSwgeSA9IFgyKSwgY29sb3IgPSBjb2xvcl9wYWxldHRlWzFdKQoKI2dncGxvdGx5KHBsdCkKCmdnc2F2ZShwbHQsIGZpbGVuYW1lID0gIi4uL2ZpZ3VyZXMvcHJfY3VydmVfZ3B0X2Z1enp5X3dhdGVybG9vLnBuZyIsIGhlaWdodCA9IDUsIHdpZHRoID0gNSkKCmBgYAoKTmV0d29yayBQcm9wZXJ0aWVzIC0gR1BUIHYgRnV6enkgdiBXYXRlcmxvbwoKCmBgYHtyfQoKcG9wdWxhdGVfbG9jYWxfbmV0d29ya19wcm9wcyA9IGZ1bmN0aW9uKGcpCnsKICBkZWcgPSBsb2coZGVncmVlKGcsIG1vZGUgPSAiYWxsIiksIDEwKQogIGRlZ1tpcy5pbmZpbml0ZShkZWcpXSA9IE5BI21pbihkZWdbIWlzLmluZmluaXRlKGRlZyldKQogIAogIGJldCA9IGxvZyhiZXR3ZWVubmVzcyhnLCBkaXJlY3RlZCA9IEZBTFNFKSwxMCkKICBiZXRbaXMuaW5maW5pdGUoYmV0KV0gPSBOQSNtaW4oYmV0WyFpcy5pbmZpbml0ZShiZXQpXSkKICAKICBlaWcgPSBsb2coZWlnZW5fY2VudHJhbGl0eShnLCBkaXJlY3RlZCA9IEZBTFNFKSR2ZWN0b3IsMTApCiAgZWlnW2lzLmluZmluaXRlKGVpZyldID0gTkEjbWluKGVpZ1shaXMuaW5maW5pdGUoZWlnKV0pCiAgCiAgY2xvID0gY2xvc2VuZXNzKGcsIG1vZGUgPSAiYWxsIikKICBjbG9baXMuaW5maW5pdGUoY2xvKV0gPSBOQSNtaW4oY2xvWyFpcy5pbmZpbml0ZShjbG8pXSkKICAKICBjb3JlID0gY29yZW5lc3MoZykKICBjb3JlW2lzLmluZmluaXRlKChjb3JlKSldID0gTkEjbWluKGNvcmVbIWlzLmluZmluaXRlKGNvcmUpXSkKICAKICB0cmFucyA9IHRyYW5zaXRpdml0eShnLCB0eXBlID0gImxvY2FsIikgCiAgdHJhbnNbaXMuaW5maW5pdGUoKHRyYW5zKSldID0gTkEjbWluKHRyYW5zWyFpcy5pbmZpbml0ZSh0cmFucyldKQogIAogIGRmID0gZGF0YS5mcmFtZSh1c2VyID0gVihnKSRuYW1lLCAKICAgICAgICAgICAgICAgICAgZGVnID0gZGVnLAogICAgICAgICAgICAgICAgICBkZWdfWiA9IChkZWctbWVhbihkZWcsIG5hLnJtID0gVFJVRSkpL3NkKGRlZywgbmEucm0gPSBUUlVFKSwKICAgICAgICAgICAgICAgICAgYmV0ID0gYmV0LAogICAgICAgICAgICAgICAgICBiZXRfWiA9IChiZXQtbWVhbihiZXQsIG5hLnJtID0gVFJVRSkpL3NkKGJldCwgbmEucm0gPSBUUlVFKSwgCiAgICAgICAgICAgICAgICAgIGNsbyA9IGNsbywKICAgICAgICAgICAgICAgICAgY2xvX1ogPSAoY2xvLW1lYW4oY2xvLCBuYS5ybSA9IFRSVUUpKS9zZChjbG8sIG5hLnJtID0gVFJVRSksIAogICAgICAgICAgICAgICAgICBlaWcgPSBlaWcsCiAgICAgICAgICAgICAgICAgIGVpZ19aID0gKGVpZy1tZWFuKGVpZywgbmEucm0gPSBUUlVFKSkvc2QoZWlnLCBuYS5ybSA9IFRSVUUpLCAKICAgICAgICAgICAgICAgICAgY29yZSA9IGNvcmUsCiAgICAgICAgICAgICAgICAgIGNvcmVfWiA9IChjb3JlLW1lYW4oY29yZSwgbmEucm0gPSBUUlVFKSkvc2QoY29yZSwgbmEucm0gPSBUUlVFKSwKICAgICAgICAgICAgICAgICAgdHJhbnMgPSB0cmFucywKICAgICAgICAgICAgICAgICAgdHJhbnNfWiA9ICh0cmFucy1tZWFuKHRyYW5zLCBuYS5ybSA9IFRSVUUpKS9zZCh0cmFucywgbmEucm0gPSBUUlVFKSkKICAKICByZXR1cm4oZGYpCn0KCnBvcHVsYXRlX2dsb2JhbF9uZXR3b3JrX3Byb3BzID0gZnVuY3Rpb24oZykKewoKICBkZiA9IGRhdGEuZnJhbWUoKQogICN3ZWIgPSBhc19hZGphY2VuY3lfbWF0cml4KGcsIHR5cGUgPSAiYm90aCIsIHNwYXJzZSA9IEZBTFNFKQogICNjb3VudHJ5ID0gYXNzb3J0bWVudC5kaXNjcmV0ZShncmFwaCA9IHdlYiwgdHlwZXMgPSBhcy5mYWN0b3IoVihnKSRDb3VudHJ5KSwgU0UgPSBUUlVFKQogIAogIGRmID0gcmJpbmQoZGYsIGRhdGEuZnJhbWUodHlwZSA9ICJlZGdlcyIsIHNjb3JlID0gZWNvdW50KGcpKSkKICBkZiA9IHJiaW5kKGRmLCBkYXRhLmZyYW1lKHR5cGUgPSAiZGVuc2l0eSIsIHNjb3JlID0gZWRnZV9kZW5zaXR5KGcpKSkKICBkZiA9IHJiaW5kKGRmLCBkYXRhLmZyYW1lKHR5cGUgPSAidHJpYW5nbGVzIiwgc2NvcmUgPSBsZW5ndGgodHJpYW5nbGVzKGcpKSkpCiAgZGYgPSByYmluZChkZiwgZGF0YS5mcmFtZSh0eXBlID0gImNvbm5lY3RlZCBjb21wb25lbnRzIiwgc2NvcmUgPSBjb21wb25lbnRzKGcpJG5vKSkKICBkZiA9IHJiaW5kKGRmLCBkYXRhLmZyYW1lKHR5cGUgPSAiYXNzb3J0YXRpdml0eV9yZWdpb24iLCBzY29yZSA9IGFzc29ydGF0aXZpdHlfbm9taW5hbChnLCBkaXJlY3RlZCA9IEZBTFNFLCB0eXBlcyA9IGFzLmZhY3RvcihWKGcpJFJlZ2lvbikpKSkKICBkZiA9IHJiaW5kKGRmLCBkYXRhLmZyYW1lKHR5cGUgPSAiYXNzb3J0YXRpdml0eV9jb3VudHJ5Iiwgc2NvcmUgPSBhc3NvcnRhdGl2aXR5X25vbWluYWwoZywgZGlyZWN0ZWQgPSBGQUxTRSwgdHlwZXMgPSBhcy5mYWN0b3IoVihnKSRDb3VudHJ5KSkpKQogICNkZiA9IHJiaW5kKGRmLCBkYXRhLmZyYW1lKHR5cGUgPSAiYXNzb3J0YXRpdml0eV90cmFjayIsIHNjb3JlID0gYXNzb3J0YXRpdml0eV9ub21pbmFsKGcsIGRpcmVjdGVkID0gRkFMU0UsIHR5cGVzID0gYXMuZmFjdG9yKFYoZykkVHJhY2spKSkpCiAgZGYgPSByYmluZChkZiwgZGF0YS5mcmFtZSh0eXBlID0gImFzc29ydGF0aXZpdHlfc2VjdGlvbiIsIHNjb3JlID0gYXNzb3J0YXRpdml0eV9ub21pbmFsKGcsIGRpcmVjdGVkID0gRkFMU0UsIHR5cGVzID0gYXMuZmFjdG9yKFYoZykkU2VjdGlvbikpKSkKICBkZiA9IHJiaW5kKGRmLCBkYXRhLmZyYW1lKHR5cGUgPSAiYXNzb3J0YXRpdml0eV9zaXplIiwgc2NvcmUgPSBhc3NvcnRhdGl2aXR5KGcsIGRpcmVjdGVkID0gRkFMU0UsIHR5cGVzMSA9IFYoZykkU2l6ZSkpKQogIGRmID0gcmJpbmQoZGYsIGRhdGEuZnJhbWUodHlwZSA9ICJhc3NvcnRhdGl2aXR5X2RlZ3JlZSIsIHNjb3JlID0gYXNzb3J0YXRpdml0eShnLCBkaXJlY3RlZCA9IEZBTFNFLCB0eXBlczEgPSBkZWdyZWUoZykpKSkKICBkZiA9IHJiaW5kKGRmLCBkYXRhLmZyYW1lKHR5cGUgPSAiYXNzb3J0YXRpdml0eV9wcmlvciBzdWNjZXNzIiwgc2NvcmUgPSBhc3NvcnRhdGl2aXR5X25vbWluYWwoZywgZGlyZWN0ZWQgPSBGQUxTRSwgdHlwZXMgPSBhcy5mYWN0b3IoVihnKSRnb2xkX3ByZXZfeWVhcikpKSkKICBkZiA9IHJiaW5kKGRmLCBkYXRhLmZyYW1lKHR5cGUgPSAiYXNzb3J0YXRpdml0eV9wcmlvciBleHBlcmllbmNlIiwgc2NvcmUgPSBhc3NvcnRhdGl2aXR5KGcsIGRpcmVjdGVkID0gRkFMU0UsIHR5cGVzMSA9IFYoZykkbmJfcGFydGljaXBhdGlvbikpKQogIAogIHJldHVybihkZikKICAKfQoKCmBgYAoKTG9jYWwgTmV0d29yayBQcm9wZXJ0aWVzCgpgYGB7cn0KCmRmX2xucCA9IGRhdGEuZnJhbWUoKQoKZGYgPSBwb3B1bGF0ZV9sb2NhbF9uZXR3b3JrX3Byb3BzKGdfd2F0ZXJsb29fMTUpCmRmID0gcmVzaGFwZTI6Om1lbHQoZGYsIGlkLnZhciA9ICJ1c2VyIikKZGYkeWVhciA9IDIwMTUKZGYkdHlwZSA9ICJ3YXRlcmxvbyIKCmRmX2xucCA9IHJiaW5kKGRmX2xucCwgZGYpCgpkZiA9IHBvcHVsYXRlX2xvY2FsX25ldHdvcmtfcHJvcHMoZ19ncHRfMTUpCmRmID0gcmVzaGFwZTI6Om1lbHQoZGYsIGlkLnZhciA9ICJ1c2VyIikKZGYkeWVhciA9IDIwMTUKZGYkdHlwZSA9ICJncHQtMy41LXR1cmJvLTE2ayIKCmRmX2xucCA9IHJiaW5kKGRmX2xucCwgZGYpCgpkZiA9IHBvcHVsYXRlX2xvY2FsX25ldHdvcmtfcHJvcHMoZ19mdXp6eV8xNSkKZGYgPSByZXNoYXBlMjo6bWVsdChkZiwgaWQudmFyID0gInVzZXIiKQpkZiR5ZWFyID0gMjAxNQpkZiR0eXBlID0gImZ1enp5LW1hdGNoaW5nIgoKZGZfbG5wID0gcmJpbmQoZGZfbG5wLCBkZikKCgpgYGAKClNjYXR0ZXJwbG90cyAKCmBgYHtyfQoKcmVuYW1lcyA9IGxpc3QoImRlZyIgPSAiTG9nMTAgRGVncmVlIiwgImRlZ19aIiA9ICJMb2cxMCBEZWdyZWUgWi1zY29yZWQiLCAiY2xvIiA9ICJDbG9zZW5lc3MiLCAiY2xvX1oiID0gIkNsb3NlbmVzcyBaLXNjb3JlZCIsICJiZXQiID0gIkxvZzEwIEJldHdlZW5uZXNzIiwgImJldF9aIiA9ICJMb2cxMCBCZXR3ZWVubmVzcyBaLXNjb3JlZCIsICJlaWciID0gIkxvZzEwIEVpZ2VuIENlbnRyYWxpdHkiLCAiZWlnX1oiID0gIkxvZzEwIEVpZ2VuIENlbnQuIFotc2NvcmVkIiwgInRyYW5zIiA9ICJMb2NhbCBDbHVzdGVyaW5nIiwgInRyYW5zX1oiID0gIkxvY2FsIENsdXN0ZXJpbmcgWi1zY29yZWQiLCAiY29yZSIgPSAiQ29yZW5lc3MiLCAiY29yZV9aIiA9ICJDb3JlIFotc2NvcmVkIikKCnBkZigiLi4vZmlndXJlcy9zY2F0dGVycGxvdHNfZ3B0X2Z1enp5X3dhdGVybG9vLnBkZiIsIGhlaWdodCA9IDcsIHdpZHRoID0gNykKCmZvciAoaSBpbiB1bmlxdWUoZGZfbG5wJHZhcmlhYmxlKSkKewogIHRlbXAgPSBkZl9sbnBbZGZfbG5wJHZhcmlhYmxlID09IGksXQogIHRlbXAgPSByZXNoYXBlMjo6ZGNhc3QodGVtcCwgZm9ybXVsYSA9IHVzZXIreWVhcn50eXBlLCB2YWx1ZS52YXIgPSAidmFsdWUiKQogIAogIAogIHBsdCA9IGdncGxvdCh0ZW1wKSArIAogICAgZ2VvbV9wb2ludChhZXMoeCA9IHdhdGVybG9vLCB5ID0gYGdwdC0zLjUtdHVyYm8tMTZrYCksIGFscGhhID0gMC44LCBjb2xvciA9IGNvbG9yX3BhbGV0dGVbM10pICsgZ2VvbV9hYmxpbmUoY29sb3IgPSAiYmxhY2siLCBhbHBoYSA9IDEpICsgCiAgICBnZW9tX3BvaW50KGFlcyh4ID0gd2F0ZXJsb28sIHkgPSBgZnV6enktbWF0Y2hpbmdgKSwgYWxwaGEgPSAwLjgsIGNvbG9yID0gY29sb3JfcGFsZXR0ZVsxXSkgKwogICAgZ2d0aXRsZSgiIikgKyAKICAgICB0aGVtZV9idyhiYXNlX3NpemUgPSAyMCkgKyAKICAgIHhsYWIoIldhdGVybG9vIikgKyB5bGFiKCJQcmVkaWN0ZWQiKSArIGdndGl0bGUocGFzdGUocmVuYW1lc1tpXSwgc2VwID0gIiIpKSArIHRoZW1lKGF4aXMudGV4dC54ID0gZWxlbWVudF90ZXh0KGFuZ2xlID0gNDUsIHZqdXN0ID0gMSwgaGp1c3Q9MSkpIAogIAogIHByaW50KHBsdCkKCn0KCmRldi5vZmYoKQoKYGBgCgpDbHVzdGVyZWQgU2NhdHRlcnBsb3RzIChUT0RPKQoKYGBge3J9CgpwZGYoIi4uL2ZpZ3VyZXMvaW50ZXJ2YWxfc2NhdHRlcnBsb3RzX2dwdF9mdXp6eV93YXRlcmxvby5wZGYiLCBoZWlnaHQgPSA3LCB3aWR0aCA9IDcpCgpmb3IgKGkgaW4gdW5pcXVlKGRmX2xucCR2YXJpYWJsZSkpCnsKICB0ZW1wID0gZGZfbG5wW2RmX2xucCR2YXJpYWJsZSA9PSBpLF0KICBicmVha3MgPC0gc2VxKG1pbih0ZW1wJHZhbHVlWyFpcy5uYSh0ZW1wJHZhbHVlKV0pLCBtYXgodGVtcCR2YWx1ZVshaXMubmEodGVtcCR2YWx1ZSldKSwgbGVuZ3RoLm91dCA9IDEwKQogIGNsYXNzZXMgPC0gY3V0KHRlbXAkdmFsdWUsIGJyZWFrcz1icmVha3MsIGluY2x1ZGUubG93ZXN0ID0gVFJVRSkKICAKICB0ZW1wJGNsYXNzID0gY2xhc3NlcwoKICAjdGVtcCA9IHJlc2hhcGUyOjpkY2FzdCh0ZW1wLCBmb3JtdWxhID0gY2xhc3N+dHlwZSwgdmFsdWUudmFyID0gInZhbHVlIikKICB0ZW1wID0gdGVtcCAlPiUgZ3JvdXBfYnkoY2xhc3MsIHR5cGUpICU+JSBzdW1tYXJpc2UobWVhbiA9IG1lYW4odmFsdWUpLCBzZSA9IHNlKHZhbHVlKSkKICB0ZW1wID0gcmVzaGFwZTI6OmRjYXN0KHRlbXAsIGZvcm11bGEgPSBjbGFzc350eXBlLCB2YWx1ZS52YXJzID0gYygibWVhbiIpKQogIAogcGx0ID0gZ2dwbG90KHRlbXApICsgCiAgICBnZW9tX3BvaW50KGFlcyh4ID0gd2F0ZXJsb28sIHkgPSBgZ3B0LTMuNS10dXJiby0xNmtgKSwgYWxwaGEgPSAwLjgsIGNvbG9yID0gY29sb3JfcGFsZXR0ZVszXSkgKyBnZW9tX2FibGluZShjb2xvciA9ICJyZWQiLCBhbHBoYSA9IDAuMykgKyAKICAgIGdlb21fcG9pbnQoYWVzKHggPSB3YXRlcmxvbywgeSA9IGBmdXp6eS1tYXRjaGluZ2ApLCBhbHBoYSA9IDAuOCwgY29sb3IgPSBjb2xvcl9wYWxldHRlWzFdKSArCiAgICBnZ3RpdGxlKCIiKSArIAogICAgIHRoZW1lX2J3KGJhc2Vfc2l6ZSA9IDIwKSArIAogICAgeGxhYigiV2F0ZXJsb28iKSArIHlsYWIoIlByZWRpY3RlZCIpICsgZ2d0aXRsZShwYXN0ZShyZW5hbWVzW2ldLCBzZXAgPSAiIikpICsgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgdmp1c3QgPSAxLCBoanVzdD0xKSkgCiAgCiAgCiAgcHJpbnQocGx0KQoKfQoKZGV2Lm9mZigpCgpgYGAKCkZhY2V0IFdyYXAgc2NhdHRlcnBsb3QKCmBgYHtyfQoKdGVtcCA9IHJlc2hhcGUyOjpkY2FzdChkZl9sbnBbZGZfbG5wJHZhcmlhYmxlICVpbiUgYygiZGVnX1oiLCAiYmV0X1oiLCAiZWlnX1oiLCAiY29yZV9aIiwgInRyYW5zX1oiKSxdLCBmb3JtdWxhID0gdmFyaWFibGUrdXNlcit5ZWFyfnR5cGUsIHZhbHVlLnZhciA9ICJ2YWx1ZSIpCgp0ZW1wID0gdGVtcCAlPiUgcm93d2lzZSgpICU+JSBtdXRhdGUocmVuYW1lZF92YXJpYWJsZSA9IHJlbmFtZXNbYXMuY2hhcmFjdGVyKHZhcmlhYmxlKV1bWzFdXSkKCnBsdCA9IGdncGxvdCh0ZW1wKSArIAogICAgZ2VvbV9wb2ludChhZXMoeCA9IHdhdGVybG9vLCB5ID0gYGdwdC0zLjUtdHVyYm8tMTZrYCksIGFscGhhID0gMC42LCBjb2xvciA9IGNvbG9yX3BhbGV0dGVbM10pICsgZ2VvbV9hYmxpbmUoY29sb3IgPSAicmVkIiwgYWxwaGEgPSAwLjEpICsgCiAgICBnZW9tX3BvaW50KGFlcyh4ID0gd2F0ZXJsb28sIHkgPSBgZnV6enktbWF0Y2hpbmdgKSwgYWxwaGEgPSAwLjMsIGNvbG9yID0gY29sb3JfcGFsZXR0ZVsxXSkgKwogICAgZ2d0aXRsZSgiIikgKyAKZmFjZXRfd3JhcCh+cmVuYW1lZF92YXJpYWJsZSkgKwogICAgICAgdGhlbWVfYncoYmFzZV9zaXplID0gMjApICsgCiAgICB4bGFiKCIiKSArIHlsYWIoIiIpICsgZ2d0aXRsZSgiIikgKyB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDQ1LCB2anVzdCA9IDEsIGhqdXN0PTEpKSAKCmdnc2F2ZShwbHQsIGZpbGVuYW1lID0gIi4uL2ZpZ3VyZXMvZmFjZXRfd3JhcF9zY2F0dGVycGxvdC5wbmciLCB3aWR0aCA9IDEwLCBoZWlnaHQgPSA4KQoKYGBgCgoKCkRlbnNpdHkgUGxvdCAoVE8gZWRpdCkKCmBgYHtyfQoKcGRmKCIuLi9maWd1cmVzL2xvY2FsX3Byb3BzX2RlbnNpdHlfMjAxNS5wZGYiLCBoZWlnaHQgPSA3LCB3aWR0aCA9IDcpCgpmb3IgKGkgaW4gdW5pcXVlKGRmX2xucCR2YXJpYWJsZSkpCnsKICB0ZW1wID0gZGZfbG5wW2RmX2xucCR2YXJpYWJsZSA9PSBpLF0KICAjdGVtcCA9IHJlc2hhcGUyOjpkY2FzdCh0ZW1wLCBmb3JtdWxhID0gdXNlcit5ZWFyfnR5cGUsIHZhbHVlLnZhciA9ICJ2YWx1ZSIpCiAgI3RlbXAgPSByZXNoYXBlMjo6ZGNhc3QodGVtcCwgZm9ybXVsYSA9IHVzZXIreWVhcn50eXBlLCB2YWx1ZS52YXIgPSAidmFsdWUiKQogIAogICNwbHQgPSBnZ3Bsb3QodGVtcCkgKwogICNnZW9tX2hpc3RvZ3JhbShhZXMoeCA9IGBncHQtMy41LXR1cmJvLTE2a2AsIHkgPSAuLmRlbnNpdHkuLiksIGZpbGw9Z2JoX3BhbGV0dGVbMV0pICsKICAjZ2VvbV9sYWJlbChhZXMoeD00LjUsIHk9MC4yNSwgbGFiZWw9IkdQVCIpLCBjb2xvcj1nYmhfcGFsZXR0ZVsxXSkgKwogICNnZW9tX2hpc3RvZ3JhbSggYWVzKHggPSBgZnV6enktbWF0Y2hpbmdgLCB5ID0gLS4uZGVuc2l0eS4uKSwgZmlsbD0gZ2JoX3BhbGV0dGVbMl0pICsKICAjZ2VvbV9sYWJlbChhZXMoeD00LjUsIHk9LTAuMjUsIGxhYmVsPSJGdXp6eSIpLCBjb2xvcj1nYmhfcGFsZXR0ZVsyXSkgKyB0aGVtZV9idyhiYXNlX3NpemUgPSAyMCkgKyAKICAjeGxhYihyZW5hbWVzW2ldKQogIAogICBwbHQgPSBnZ3Bsb3QodGVtcCkgKyBnZW9tX2RlbnNpdHkoYWVzKHggPSB2YWx1ZSwgZ3JvdXAgPSB0eXBlLCBmaWxsID0gdHlwZSwgY29sb3IgPSB0eXBlKSwgYWxwaGEgPSAwLjgpICsKICAgIGdndGl0bGUoIiIpICsgc2NhbGVfZmlsbF9tYW51YWwoYnJlYWtzID0gYygiZ3B0LTMuNS10dXJiby0xNmsiLCAiZnV6enktbWF0Y2hpbmciLCAid2F0ZXJsb28iKSwgdmFsdWVzID0gY29sb3JfcGFsZXR0ZVtjKDEsNCw2KV0pICsKICAgICBzY2FsZV9jb2xvcl9tYW51YWwoYnJlYWtzID0gYygiZ3B0LTMuNS10dXJiby0xNmsiLCAiZnV6enktbWF0Y2hpbmciLCAid2F0ZXJsb28iKSwgdmFsdWVzID0gY29sb3JfcGFsZXR0ZVtjKDEsNCw2KV0pICsKICAgICB0aGVtZV9idyhiYXNlX3NpemUgPSAyMCkgKyAKICAgIHhsYWIoIiIpICsgeWxhYigiIikgKyBnZ3RpdGxlKHBhc3RlKHJlbmFtZXNbaV0sIHNlcCA9ICIiKSkgKyB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDQ1LCB2anVzdCA9IDEsIGhqdXN0PTEpKSAKICAKICBwcmludChwbHQpCgp9CgpkZXYub2ZmKCkKCmBgYAoKRGVncmVlIERpc3RyaWJ1dGlvbgoKCjIwMTU6IEdQVCB2IEZ1enp5IHYgV2F0ZXJsb28KCmBgYHtyfQoKZGVncmVlX2Rpc3RyaWJ1dGlvbl9ub3JtYWxpc2VkID0gZnVuY3Rpb24gKGdyYXBoLCBjdW11bGF0aXZlID0gRkFMU0UpIAp7CiAgaWYgKCFpc19pZ3JhcGgoZ3JhcGgpKSB7CiAgICBzdG9wKCJOb3QgYSBncmFwaCBvYmplY3QiKQogIH0KICBjcyA8LSBkZWdyZWUoZ3JhcGgpCiAgY3MgPSAoY3MpL21heChjcykKICAKICBoaSA8LSBoaXN0KGNzLCBwbG90ID0gRkFMU0UpJGRlbnNpdHkKICBpZiAoIWN1bXVsYXRpdmUpIHsKICAgIHJlcyA8LSBoaQogIH0KICBlbHNlIHsKICAgIHJlcyA8LSByZXYoY3Vtc3VtKHJldihoaSkpKQogIH0KICByZXR1cm4ocmVzKQp9CgoKYGBgCgoKYGBge3J9CgpkZl9jdW0gPSBkYXRhLmZyYW1lKCkKCmdfZ3B0ID0gZ19ncHRfMTUKZ19mdXp6eSA9IGdfZnV6enlfMTUKICAKZGlzdCA9IGxvZyhkZWdyZWVfZGlzdHJpYnV0aW9uX25vcm1hbGlzZWQoZ19ncHQsIGN1bXVsYXRpdmUgPSBUUlVFKSwgYmFzZT0gMTApCmRmX2N1bSA9IHJiaW5kKGRmX2N1bSwgZGF0YS5mcmFtZShuZXR3b3JrID0gImdwdC0zLjUtdHVyYm8tMTZrIiwgZGVncmVlID0gMTpsZW5ndGgoZGlzdCksIGRlZ3JlZV9sb2cgPSBsb2coMTpsZW5ndGgoZGlzdCksIGJhc2UgPSAxMCksIHByb2IgPSBkaXN0LCB5ZWFyID0gaSkpCgpkaXN0ID0gbG9nKGRlZ3JlZV9kaXN0cmlidXRpb25fbm9ybWFsaXNlZChnX2Z1enp5LCBjdW11bGF0aXZlID0gVFJVRSksIGJhc2U9IDEwKQpkZl9jdW0gPSByYmluZChkZl9jdW0sIGRhdGEuZnJhbWUobmV0d29yayA9ICJmdXp6eS1tYXRjaGluZyIsIGRlZ3JlZSA9IDE6bGVuZ3RoKGRpc3QpLCBkZWdyZWVfbG9nID0gbG9nKDE6bGVuZ3RoKGRpc3QpLCBiYXNlID0gMTApLCBwcm9iID0gZGlzdCwgeWVhciA9IGkpKQogIApkaXN0ID0gbG9nKGRlZ3JlZV9kaXN0cmlidXRpb25fbm9ybWFsaXNlZChnX3dhdGVybG9vXzE1LCBjdW11bGF0aXZlID0gVFJVRSksIGJhc2U9IDEwKQpkZl9jdW0gPSByYmluZChkZl9jdW0sIGRhdGEuZnJhbWUobmV0d29yayA9ICJ3YXRlcmxvbyIsIGRlZ3JlZSA9IDE6bGVuZ3RoKGRpc3QpLCBkZWdyZWVfbG9nID0gbG9nKDE6bGVuZ3RoKGRpc3QpLCBiYXNlID0gMTApLCBwcm9iID0gZGlzdCwgeWVhciA9IGkpKQogIApwbHQgPSBnZ3Bsb3QoZGZfY3VtLCBhZXMoY29sb3IgPSBuZXR3b3JrLCB5PXByb2IsIHg9IGRlZ3JlZV9sb2cpKSArIAogICAgZ2VvbV9saW5lKHNpemUgPSAxLjIpICsKICAgIHNjYWxlX2NvbG9yX21hbnVhbChicmVha3MgPSBjKCJncHQtMy41LXR1cmJvLTE2ayIsICJmdXp6eS1tYXRjaGluZyIsICJ3YXRlcmxvbyIpLCB2YWx1ZXMgPSBjb2xvcl9wYWxldHRlW2MoMSwzLDYpXSkgKwogICAgZ2d0aXRsZSgiIikgKwogICAgICBmYWNldF93cmFwKH4iMjAxNSIpICsgdGhlbWVfYncoYmFzZV9zaXplID0gMjApICsgCiAgICB4bGFiKCIiKSArIHlsYWIoIiIpICsgZ2d0aXRsZSgiIikKCiN0aGVtZShwYW5lbC5ncmlkLm1ham9yID0gZWxlbWVudF9ibGFuaygpLCBwYW5lbC5ncmlkLm1pbm9yID0gZWxlbWVudF9ibGFuaygpLCBwYW5lbC5iYWNrZ3JvdW5kID0gZWxlbWVudF9ibGFuaygpKQoKZ2dzYXZlKHBsdCwgZmlsZW5hbWUgPSAiLi4vZmlndXJlcy9jdW11bGF0aXZlX2RkXzIwMTUucG5nIiwgd2lkdGggPSAxMCwgaGVpZ2h0ID0gOCkKCmBgYAoKQ29ycmVsYXRpb25zIEJveHBsb3RzCgpDb3JyZWxhdGlvbnMgMjAxNSAtIEdQVCB2IFdhdGVybG9vICYgRnV6enkgdiBXYXRlcmxvbwoKYGBge3IsIHdhcm5pbmc9RkFMU0UsIG1lc3NhZ2U9RkFMU0V9CgpkZl9sbnBfY29ycmVsYXRpb25zID0gZGF0YS5mcmFtZSgpCgpmb3IgKGkgaW4gdW5pcXVlKGRmX2xucCR2YXJpYWJsZSkpCnsKICB0ZW1wID0gZGZfbG5wW2RmX2xucCR2YXJpYWJsZSA9PSBpLF0KICB0ZW1wID0gcmVzaGFwZTI6OmRjYXN0KHRlbXAsIGZvcm11bGEgPSB1c2VyK3llYXJ+dHlwZSwgdmFsdWUudmFyID0gInZhbHVlIikKCiAgY29yID0gY29yLnRlc3QodGVtcCRgZ3B0LTMuNS10dXJiby0xNmtgLCB0ZW1wJHdhdGVybG9vLCBtZXRob2QgPSAic3BlYXJtYW4iKQogIGRmX2xucF9jb3JyZWxhdGlvbnMgPSByYmluZChkZl9sbnBfY29ycmVsYXRpb25zLCBkYXRhLmZyYW1lKHR5cGUgPSAiZ3B0LTMuNS10dXJiby0xNmsiLCB2YXJpYWJsZSA9IGksIGNvciA9IGNvciRlc3RpbWF0ZSwgcF92YWwgPSBjb3IkcC52YWx1ZSkpCiAgCiAgY29yID0gY29yLnRlc3QodGVtcCRgZnV6enktbWF0Y2hpbmdgLCB0ZW1wJHdhdGVybG9vLCBtZXRob2QgPSAic3BlYXJtYW4iKSAgCiAgZGZfbG5wX2NvcnJlbGF0aW9ucyA9IHJiaW5kKGRmX2xucF9jb3JyZWxhdGlvbnMsIGRhdGEuZnJhbWUodHlwZSA9ICJmdXp6eS1tYXRjaGluZyIsIHZhcmlhYmxlID0gaSwgY29yID0gY29yJGVzdGltYXRlLCBwX3ZhbCA9IGNvciRwLnZhbHVlKSkKfQoKYGBgCgoKCkJveHBsb3RzIChUTyBDSEVDSykKCmBgYHtyfQoKcGx0ID0gZ2dwbG90KGRmX2xucF9jb3JyZWxhdGlvbnMsIGFlcyh4ID0gY29yLCB5ID0gdmFyaWFibGUpKSArIAogICAgZ2VvbV9iYXIoc3RhdCA9ICJpZGVudGl0eSIpICsgc2NhbGVfeV9kaXNjcmV0ZShsYWJlbHMgPSByZW5hbWVzKSArIAogICAgZ2d0aXRsZSgiIikgKyAKIHRoZW1lX2J3KGJhc2Vfc2l6ZSA9IDIwKSArCiAgICB4bGFiKCJTcGVhcm1hbiBDb3JyZWxhdGlvbnMiKSArIHlsYWIoIiIpICsgZ2d0aXRsZSgiIikgCgojZ2dzYXZlKHBsdCwgZmlsZW5hbWUgPSAiLi4vZmlndXJlcy9jb3JyZWxhdGlvbnNfeWVhcnMucG5nIiwgaGVpZ2h0ID0gNywgd2lkdGggPSA5KQoKYGBgCgpHbG9iYWwgTmV0d29yayBQcm9wZXJ0aWVzCgpgYGB7cn0KCmRmX2ducCA9IGRhdGEuZnJhbWUoKQoKI0dQVApkZiA9IHBvcHVsYXRlX2dsb2JhbF9uZXR3b3JrX3Byb3BzKGdfZ3B0XzE1KQpkZiRtb2RlbCA9ICJncHQtMy41LXR1cmJvLTE2ayIKZGZfZ25wID0gcmJpbmQoZGZfZ25wLCBkZikKCiNmdXp6eQpkZiA9IHBvcHVsYXRlX2dsb2JhbF9uZXR3b3JrX3Byb3BzKGdfZnV6enlfMTUpCmRmJG1vZGVsID0gImZ1enp5LW1hdGNoaW5nIgpkZl9nbnAgPSByYmluZChkZl9nbnAsIGRmKQoKI3dhdGVybG9vCmRmID0gcG9wdWxhdGVfZ2xvYmFsX25ldHdvcmtfcHJvcHMoZ193YXRlcmxvb18xNSkKZGYkbW9kZWwgPSAid2F0ZXJsb28iCmRmX2ducCA9IHJiaW5kKGRmX2ducCwgZGYpCgoKYGBgCgpCYXIgUGxvdCAobWludXMgamFja2tuaWZlKQoKYGBge3J9CgpwbHQgPSBnZ3Bsb3QoZGZfZ25wWyFkZl9nbnAkdHlwZSAlaW4lIGMoImVkZ2VzIiwgInRyaWFuZ2xlcyIsICJjb25uZWN0ZWQgY29tcG9uZW50cyIpLF0pICsgZ2VvbV9iYXIoYWVzKHggPSB0eXBlLCB5ID0gc2NvcmUsIGZpbGwgPSBtb2RlbCksIHN0YXQgPSAiaWRlbnRpdHkiLCBwb3NpdGlvbiA9IHBvc2l0aW9uX2RvZGdlKCkpICsgdGhlbWVfYncoYmFzZV9zaXplID0gMjUpICsgc2NhbGVfZmlsbF9tYW51YWwoYnJlYWtzID0gYygiZ3B0LTMuNS10dXJiby0xNmsiLCAiZnV6enktbWF0Y2hpbmciLCAid2F0ZXJsb28iKSwgdmFsdWVzID0gY29sb3JfcGFsZXR0ZVtjKDEsNCw2KV0pICsgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgdmp1c3QgPSAxLCBoanVzdD0xKSkKCmBgYAoKKioqCgpDb3JyZWxhdGlvbnMgd2l0aCBKYWNra25pZmVzCgooTE9DQUwpCgpgYGB7cn0KIyAKIyBkZl9sY29yX2prbiA9IGRhdGEuZnJhbWUoKQojIAojIGZvciAobmFtZSBpbiBWKGdfd2F0ZXJsb29fMTUpJG5hbWUpCiMgewojICAgdGVtcCA9IGRhdGEuZnJhbWUoKQojICAgCiMgICBnX2dwdCA9IGRlbGV0ZS52ZXJ0aWNlcyhnX2dwdF8xNSwgdiA9IG5hbWUpCiMgICBnX2Z1enp5ID0gZGVsZXRlLnZlcnRpY2VzKGdfZnV6enlfMTUsIHYgPSBuYW1lKQojICAgZ193YXRlcmxvbyA9IGRlbGV0ZS52ZXJ0aWNlcyhnX3dhdGVybG9vXzE1LCB2ID0gbmFtZSkKIyAgIAojICAgZGYgPSBwb3B1bGF0ZV9sb2NhbF9uZXR3b3JrX3Byb3BzKGdfZ3B0KQojICAgZGYgPSByZXNoYXBlMjo6bWVsdChkZiwgaWQudmFyID0gInVzZXIiKQojICAgZGYkbW9kZWwgPSAiZ3B0LTMuNS10dXJiby0xNmsiCiMgICB0ZW1wID0gcmJpbmQodGVtcCwgZGYpCiMgCiMgICBkZiA9IHBvcHVsYXRlX2xvY2FsX25ldHdvcmtfcHJvcHMoZ19mdXp6eSkKIyAgIGRmID0gcmVzaGFwZTI6Om1lbHQoZGYsIGlkLnZhciA9ICJ1c2VyIikKIyAgIGRmJG1vZGVsID0gImZ1enp5LW1hdGNoaW5nIgojICAgdGVtcCA9IHJiaW5kKHRlbXAsIGRmKQojIAojICAgZGYgPSBwb3B1bGF0ZV9sb2NhbF9uZXR3b3JrX3Byb3BzKGdfd2F0ZXJsb28pCiMgICBkZiA9IHJlc2hhcGUyOjptZWx0KGRmLCBpZC52YXIgPSAidXNlciIpCiMgICBkZiRtb2RlbCA9ICJ3YXRlcmxvbyIKIyAgIHRlbXAgPSByYmluZCh0ZW1wLCBkZikKIyAgIAojICAgdGVtcCRyZW1vdmVkID0gbmFtZQojICAgCiMgICBkZl9sY29yX2prbiA9IHJiaW5kKGRmX2xjb3JfamtuLCB0ZW1wKQojICAgCiMgfQoKYGBgCgoKQ29tcHV0ZSBFcnJvcmJhcnMgd2l0aCBKYWNra25pZmUKCmBgYHtyLCB3YXJuaW5nPUZBTFNFfQoKIyB0ZW1wID0gZGZfbGNvcl9qa24KIyB0ZW1wID0gcmVzaGFwZTI6OmRjYXN0KHRlbXAsIGZvcm11bGEgPSB1c2VyK3JlbW92ZWQrdmFyaWFibGV+bW9kZWwsIHZhbHVlLnZhciA9ICJ2YWx1ZSIpCiMgICAKIyB0ID0gdGVtcCAlPiUgZ3JvdXBfYnkocmVtb3ZlZCwgdmFyaWFibGUpICU+JSBzdW1tYXJpc2UoY29yX2dwdCA9IGNvci50ZXN0KHdhdGVybG9vLCBgZ3B0LTMuNS10dXJiby0xNmtgLCBtZXRob2QgPSAic3BlYXJtYW4iKSRlc3RpbWF0ZSwgY29yX2Z1enp5ID0gY29yLnRlc3Qod2F0ZXJsb28sIGBmdXp6eS1tYXRjaGluZ2AsIG1ldGhvZCA9ICJzcGVhcm1hbiIpJGVzdGltYXRlKQoKZGZfY29yX2prbiA9IGRhdGEuZnJhbWUoKQoKZm9yIChpIGluIHVuaXF1ZShkZl9sbnAkdmFyaWFibGUpKQp7CiAgCiAgdGVtcCA9IGRmX2xucFtkZl9sbnAkdmFyaWFibGUgPT0gaSxdCiAgdGVtcCA9IHJlc2hhcGUyOjpkY2FzdCh0ZW1wLCBmb3JtdWxhID0gdXNlcit5ZWFyfnR5cGUsIHZhbHVlLnZhciA9ICJ2YWx1ZSIpCiAgCiAgamtuID0gZGF0YS5mcmFtZSgpCiAgCiAgZm9yIChqIGluIDE6bnJvdyh0ZW1wKSkKICB7CiAgICB0ID0gdGVtcFstYyhqKSxdCiAgICAKICAgIGprbiA9IHJiaW5kKGprbiwgZGF0YS5mcmFtZShpZCA9IGosIGNvcl9ncHQgPSBjb3IudGVzdCh0JGBncHQtMy41LXR1cmJvLTE2a2AsIHQkd2F0ZXJsb28sIG1ldGhvZCA9ICJzcGVhcm1hbiIpJGVzdGltYXRlLCBjb3JfZnV6enkgPSBjb3IudGVzdCh0JGBmdXp6eS1tYXRjaGluZ2AsIHQkd2F0ZXJsb28sIG1ldGhvZCA9ICJzcGVhcm1hbiIpJGVzdGltYXRlKSkKICAKICB9CiAgCiAgYWN0dWFsID0gY29yLnRlc3QodGVtcCRgZ3B0LTMuNS10dXJiby0xNmtgLCB0ZW1wJHdhdGVybG9vLCBtZXRob2QgPSAic3BlYXJtYW4iKSRlc3RpbWF0ZQogIGVzdGltYXRlcyA9IGprbiRjb3JfZ3B0CiAgCiAgZGZfY29yX2prbiA9IHJiaW5kKGRmX2Nvcl9qa24sIGRhdGEuZnJhbWUodmFyaWFibGUgPSBpLCB2YWx1ZSA9IGFjdHVhbCwgc2UgPSBqYWNra25pZmVfc2UoZXN0aW1hdGVzLCBhY3R1YWwpLCB0eXBlID0gImdwdC0zLjUtdHVyYm8tMTZrIikpCiAgCiAgYWN0dWFsID0gY29yLnRlc3QodGVtcCRgZnV6enktbWF0Y2hpbmdgLCB0ZW1wJHdhdGVybG9vLCBtZXRob2QgPSAic3BlYXJtYW4iKSRlc3RpbWF0ZQogIGVzdGltYXRlcyA9IGprbiRjb3JfZnV6enkKICAKICBkZl9jb3JfamtuID0gcmJpbmQoZGZfY29yX2prbiwgZGF0YS5mcmFtZSh2YXJpYWJsZSA9IGksIHZhbHVlID0gYWN0dWFsLCBzZSA9IGphY2trbmlmZV9zZShlc3RpbWF0ZXMsIGFjdHVhbCksIHR5cGUgPSAiZnV6enktbWF0Y2hpbmciKSkKICAKfQoKYGBgCgpNYWtlIHRoZSBiYXIgcGxvdHMKCmBgYHtyfQojW2RmX2Nvcl9qa24kdmFyaWFibGUgJWluJSBjKCJkZWciLCAiY29yZSIsICJiZXQiLCAiY2xvIiwgImVpZyIpLF0KCgpyZW5hbWVzID0gYygiY29yZSIgPSAiQ29yZW5lc3MiLCAiYmV0IiA9ICJCZXR3ZWVubmVzcyIsICJjbG8iID0gIkNsb3NlbmVzcyIsICJ0cmFucyIgPSAiQ2x1c3RlcmluZyIpCgpwbHQgPSBnZ3Bsb3QoZGZfY29yX2prbltkZl9jb3JfamtuJHZhcmlhYmxlICVpbiUgYygiY29yZSIsICJiZXQiLCAiY2xvIiwgInRyYW5zIiksXSkgKyAKICAgIGdlb21fYmFyKGFlcyhmaWxsID0gdHlwZSwgeT12YWx1ZSwgeD12YXJpYWJsZSksIHBvc2l0aW9uPSJkb2RnZSIsIHN0YXQ9ImlkZW50aXR5IikgKwogICAgc2NhbGVfZmlsbF9tYW51YWwoYnJlYWtzID0gIGMoImZ1enp5LW1hdGNoaW5nIiwgICJncHQtMy41LXR1cmJvLTE2ayIpLCB2YWx1ZXMgPSAoY29sb3JfcGFsZXR0ZVtjKDEsMyldKSkgKwogICAgZ2VvbV9lcnJvcmJhcihhZXMoZmlsbCA9IHR5cGUsIHltaW4gPSB2YWx1ZS1zZSwgeW1heCA9IHZhbHVlK3NlLCB4ID0gdmFyaWFibGUpLCB3aWR0aCA9IDAuMSwgcG9zaXRpb24gPSBwb3NpdGlvbl9kb2RnZSgwLjkpKSArCiAgICBnZ3RpdGxlKCIiKSArCiAgICB0aGVtZV9idyhiYXNlX3NpemUgPSAyNSkgKyBjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoLTAuMjUsIDEpKSArIAogICAgeGxhYigiIikgKyB5bGFiKCIiKSArIHRoZW1lKHBhbmVsLmdyaWQubWFqb3IgPSBlbGVtZW50X2JsYW5rKCksCnBhbmVsLmdyaWQubWlub3IgPSBlbGVtZW50X2JsYW5rKCkpICsgdGhlbWUobGVnZW5kLnBvc2l0aW9uPSJub25lIikgKyBzY2FsZV94X2Rpc2NyZXRlKGxhYmVscyA9IHJlbmFtZXMpICsgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgdmp1c3QgPSAxLCBoanVzdD0xKSkKCiNnZ3Bsb3RseShwbHQpCgpnZ3NhdmUocGx0LCBmaWxlbmFtZSA9ICIuLi9maWd1cmVzL2xvY2FsX25ldF9wcm9wc19ncHRfZnV6enlfd2F0ZXJsb28ucG5nIiwgd2lkdGggPSA0LCBoZWlnaHQgPSA2KQoKYGBgCgooR0xPQkFMIE5FVFdPUksgUFJPUEVSVElFUykKCmBgYHtyfQoKZGZfamtuX2ducCA9IGRhdGEuZnJhbWUoKQoKZm9yIChuYW1lIGluIFYoZ193YXRlcmxvb18xNSkkbmFtZSkKewogIGRmX2ducCA9IGRhdGEuZnJhbWUoKQogCiAgZ19ncHQgPSBkZWxldGUudmVydGljZXMoZ19ncHRfMTUsIHYgPSBuYW1lKQogIGdfZnV6enkgPSBkZWxldGUudmVydGljZXMoZ19mdXp6eV8xNSwgdiA9IG5hbWUpCiAgZ193YXRlcmxvbyA9IGRlbGV0ZS52ZXJ0aWNlcyhnX3dhdGVybG9vXzE1LCB2ID0gbmFtZSkKCiAgI0dQVAogIGRmID0gcG9wdWxhdGVfZ2xvYmFsX25ldHdvcmtfcHJvcHMoZ19ncHQpCiAgZGYkbW9kZWwgPSAiZ3B0LTMuNS10dXJiby0xNmsiCiAgZGZfZ25wID0gcmJpbmQoZGZfZ25wLCBkZikKCiNmdXp6eQogIGRmID0gcG9wdWxhdGVfZ2xvYmFsX25ldHdvcmtfcHJvcHMoZ19mdXp6eSkKICBkZiRtb2RlbCA9ICJmdXp6eS1tYXRjaGluZyIKICBkZl9nbnAgPSByYmluZChkZl9nbnAsIGRmKQoKI3dhdGVybG9vCiAgZGYgPSBwb3B1bGF0ZV9nbG9iYWxfbmV0d29ya19wcm9wcyhnX3dhdGVybG9vKQogIGRmJG1vZGVsID0gIndhdGVybG9vIgogIGRmX2ducCA9IHJiaW5kKGRmX2ducCwgZGYpCiAgCiAgZGZfZ25wJHJlbW92ZWQgPSBuYW1lCiAgCiAgZGZfamtuX2ducCA9IHJiaW5kKGRmX2prbl9nbnAsIGRmX2ducCkKfQoKZGZfZ25wID0gZGF0YS5mcmFtZSgpCiNHUFQKZGYgPSBwb3B1bGF0ZV9nbG9iYWxfbmV0d29ya19wcm9wcyhnX2dwdF8xNSkKZGYkbW9kZWwgPSAiZ3B0LTMuNS10dXJiby0xNmsiCmRmX2ducCA9IHJiaW5kKGRmX2ducCwgZGYpCgojZnV6enkKZGYgPSBwb3B1bGF0ZV9nbG9iYWxfbmV0d29ya19wcm9wcyhnX2Z1enp5XzE1KQpkZiRtb2RlbCA9ICJmdXp6eS1tYXRjaGluZyIKZGZfZ25wID0gcmJpbmQoZGZfZ25wLCBkZikKCiN3YXRlcmxvbwpkZiA9IHBvcHVsYXRlX2dsb2JhbF9uZXR3b3JrX3Byb3BzKGdfd2F0ZXJsb29fMTUpCmRmJG1vZGVsID0gIndhdGVybG9vIgpkZl9nbnAgPSByYmluZChkZl9nbnAsIGRmKQoKYGBgCgpBc3NvcnRhdGl2aXR5IAoKYGBge3J9CgpkZl9nbnBfYXNzb3J0ID0gZGF0YS5mcmFtZSgpCgpmb3IgKGkgaW4gdW5pcXVlKGRmX2ducCR0eXBlKSkKewogIHRlbXAgPSBkZl9nbnBbZGZfZ25wJHR5cGUgPT0gaSxdCiAgCiAgCiAgI3RlbXAgPSByZXNoYXBlMjo6ZGNhc3QodGVtcCwgZm9ybXVsYSA9IG1vZGVsfnNjb3JlLCB2YWx1ZS52YXIgPSAic2NvcmUiKQogIHZhbCA9IHRlbXAkc2NvcmVbdGVtcCRtb2RlbCA9PSAiZ3B0LTMuNS10dXJiby0xNmsiXQogIAogIGRmX2ducF9hc3NvcnQgPSByYmluZChkZl9nbnBfYXNzb3J0LCBkYXRhLmZyYW1lKHR5cGUgPSAiZ3B0LTMuNS10dXJiby0xNmsiLCB2YXJpYWJsZSA9IGksIHZhbCA9IHZhbCwgc2UgPSBqYWNra25pZmVfc2UoZGZfamtuX2ducCRzY29yZVtkZl9qa25fZ25wJG1vZGVsID09ICJncHQtMy41LXR1cmJvLTE2ayIgJiBkZl9qa25fZ25wJHR5cGUgPT0gaV0sIHZhbCkpKQogIAogICNjb3IgPSBjb3IudGVzdCh0ZW1wJGBmdXp6eS1tYXRjaGluZ2AsIHRlbXAkd2F0ZXJsb28sIG1ldGhvZCA9ICJzcGVhcm1hbiIpICAKICAKICB2YWwgPSB0ZW1wJHNjb3JlW3RlbXAkbW9kZWwgPT0gImZ1enp5LW1hdGNoaW5nIl0KICAKICBkZl9nbnBfYXNzb3J0ID0gcmJpbmQoZGZfZ25wX2Fzc29ydCwgZGF0YS5mcmFtZSh0eXBlID0gImZ1enp5LW1hdGNoaW5nIiwgdmFyaWFibGUgPSBpLCB2YWwgPSB2YWwsIHNlID0gamFja2tuaWZlX3NlKGRmX2prbl9nbnAkc2NvcmVbZGZfamtuX2ducCRtb2RlbCA9PSAiZnV6enktbWF0Y2hpbmciICYgZGZfamtuX2ducCR0eXBlID09IGldLCB2YWwpKSkKICAKICAKICB2YWwgPSB0ZW1wJHNjb3JlW3RlbXAkbW9kZWwgPT0gIndhdGVybG9vIl0KICAKICBkZl9nbnBfYXNzb3J0ID0gcmJpbmQoZGZfZ25wX2Fzc29ydCwgZGF0YS5mcmFtZSh0eXBlID0gIndhdGVybG9vIiwgdmFyaWFibGUgPSBpLCB2YWwgPSB2YWwsIHNlID0gamFja2tuaWZlX3NlKGRmX2prbl9nbnAkc2NvcmVbZGZfamtuX2ducCRtb2RlbCA9PSAid2F0ZXJsb28iICYgZGZfamtuX2ducCR0eXBlID09IGldLCB2YWwpKSkKICAKfQoKYGBgCgpQbG90CgpgYGB7cn0KCnJlbmFtZXMgPSBsaXN0KCJhc3NvcnRhdGl2aXR5X2NvdW50cnkiID0gIkNvdW50cnkiLCAiYXNzb3J0YXRpdml0eV9yZWdpb24iID0gIlJlZ2lvbiIsICJhc3NvcnRhdGl2aXR5X2RlZ3JlZSIgPSAiRGVncmVlIiwgImFzc29ydGF0aXZpdHlfcHJpb3IgZXhwZXJpZW5jZSIgPSAiUHJpb3IgRXhwZXJpZW5jZSIsICJhc3NvcnRhdGl2aXR5X3ByaW9yIHN1Y2Nlc3MiID0gIlN1Y2Nlc3MiLCAiYXNzb3J0YXRpdml0eV9zaXplIiA9ICJTaXplIiwgImNvbm5lY3RlZCBjb21wb25lbnRzIiA9ICJDb21wb25lbnRzIiwgInRyaWFuZ2xlcyIgPSAiVHJpYW5nbGVzIiwgImRlbnNpdHkiID0gIkRlbnNpdHkiLCAiZWRnZXMiID0gIkVkZ2VzIiwgImFzc29ydGF0aXZpdHlfc2VjdGlvbiIgPSAiU2VjdGlvbiIpCgpwbHQgPSBnZ3Bsb3QoZGZfZ25wX2Fzc29ydFtkZl9nbnBfYXNzb3J0JHZhcmlhYmxlICVpbiUgYygiYXNzb3J0YXRpdml0eV9jb3VudHJ5IiwgImFzc29ydGF0aXZpdHlfcmVnaW9uIiwgImFzc29ydGF0aXZpdHlfc2VjdGlvbiIsICJhc3NvcnRhdGl2aXR5X2RlZ3JlZSIpLF0pICsgCiAgICBnZW9tX2JhcihhZXMoZmlsbCA9IHR5cGUsIHk9dmFsLCB4PXZhcmlhYmxlKSwgcG9zaXRpb249ImRvZGdlIiwgc3RhdD0iaWRlbnRpdHkiKSArCiAgICBzY2FsZV9maWxsX21hbnVhbChicmVha3MgPSAgYygiZnV6enktbWF0Y2hpbmciLCAgImdwdC0zLjUtdHVyYm8tMTZrIiwgIndhdGVybG9vIiksIHZhbHVlcyA9IGMoY29sb3JfcGFsZXR0ZVtjKDEsMyldLCAieWVsbG93MyIpKSArCiAgICBnZW9tX2Vycm9yYmFyKGFlcyhmaWxsID0gdHlwZSwgeW1pbiA9IHZhbC1zZSwgeW1heCA9IHZhbCtzZSwgeCA9IHZhcmlhYmxlKSwgd2lkdGggPSAwLjEsIHBvc2l0aW9uID0gcG9zaXRpb25fZG9kZ2UoMC45KSkgKwogICAgZ2d0aXRsZSgiIikgKwogICAgdGhlbWVfYncoYmFzZV9zaXplID0gMjUpICsgY29vcmRfY2FydGVzaWFuKHlsaW0gPSBjKC0wLjI1LCAxKSkgKyAKICAgIHhsYWIoIiIpICsgeWxhYigiIikgKyB0aGVtZShwYW5lbC5ncmlkLm1ham9yID0gZWxlbWVudF9ibGFuaygpLApwYW5lbC5ncmlkLm1pbm9yID0gZWxlbWVudF9ibGFuaygpKSArIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbj0ibm9uZSIpICsgc2NhbGVfeF9kaXNjcmV0ZShsYWJlbHMgPSByZW5hbWVzKSArIHRoZW1lKGF4aXMudGV4dC54ID0gZWxlbWVudF90ZXh0KGFuZ2xlID0gNDUsIHZqdXN0ID0gMSwgaGp1c3Q9MSkpICsgZ2d0aXRsZSgiIikKCmdnc2F2ZShwbHQsIGZpbGVuYW1lID0gIi4uL2ZpZ3VyZXMvYXNzb3J0YXRpdml0eV8yMDE1LnBuZyIsIHdpZHRoID0gNSwgaGVpZ2h0ID0gNikKCgpgYGAKCg==